def test_filter_contains_not___value_is_lookup(runner_class): input_data = [ { "a": 1, "b": 2, "c": 1 }, { "a": 3, "b": 4, "c": 4 }, { "a": 5, "b": 6, "c": 5 }, { "a": 7, "b": 8, "c": 8 }, ] mapping = make_simple_mapping({ "c": [TransformationEntry( transformation="a * 2", when="not (a is 1)", )], "d": [TransformationEntry( transformation="b + 3", when="not (b is 6)", )], }) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": NotSet, "d": 5 }, { "c": 6, "d": 7 }, { "c": 10, "d": NotSet }, { "c": 14, "d": 11 }, ]
def test_when_contains_search_on_non_lookup(runner_class): input_data = [ {"a": "foo a", "b": "foo b"}, {"a": "far a", "b": "far b"}, {"a": "boo a", "b": "boo b"}, {"a": "bar a", "b": "bar b"}, ] mapping = make_simple_mapping( { "c": [ TransformationEntry( transformation="a", when="search('a foo b', 'foo')", ) ], "d": [ TransformationEntry( transformation="a + ' ' + b", when=r"search('foo a bar', re'oo.\w')", ) ], } ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"c": "foo a", "d": "foo a foo b"}, {"c": "far a", "d": "far a far b"}, {"c": "boo a", "d": "boo a boo b"}, {"c": "bar a", "d": "bar a bar b"}, ]
def test_transform_contains_replace_on_non_lookup(runner_class): input_data = [ {"a": "foo a", "b": "foo b"}, {"a": "far a", "b": "far b"}, {"a": "boo a", "b": "boo b"}, {"a": "bar a", "b": "bar b"}, ] mapping = make_simple_mapping( { "c": [ TransformationEntry( transformation="replace('foo', 'oo', 'aa')", ) ], "d": [ TransformationEntry( transformation=(r"replace('foo, bee', re'o{2}', 'aa')"), ) ], } ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"c": "faa", "d": "faa, bee"}, {"c": "faa", "d": "faa, bee"}, {"c": "faa", "d": "faa, bee"}, {"c": "faa", "d": "faa, bee"}, ]
def test_when_contains_search(runner_class): input_data = [ {"a": "foo a", "b": "foo b"}, {"a": "far a", "b": "far b"}, {"a": "boo a", "b": "boo b"}, {"a": "bar a", "b": "bar b"}, ] mapping = make_simple_mapping( { "c": [ TransformationEntry( transformation="a", when="search(a, 'far')", ) ], "d": [ TransformationEntry( transformation="a + ' ' + b", when=r"search(a + ' ' + b, re'a\s\w+oo')", ) ], } ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"c": NotSet, "d": "foo a foo b"}, {"c": "far a", "d": NotSet}, {"c": NotSet, "d": "boo a boo b"}, ]
def test_transform_contains_replace(runner_class): input_data = [ {"a": "foo a", "b": "foo b"}, {"a": "far a", "b": "far b"}, {"a": "boo a", "b": "boo b"}, {"a": "bar a", "b": "bar b"}, ] mapping = make_simple_mapping( { "c": [ TransformationEntry(transformation="replace(a, 'oo', 'aa')",) ], "d": [ TransformationEntry( transformation=( r"replace(a + ' ' + b, re'oo (.)', '\1\1')" ), ) ], } ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"c": "faa a", "d": "faa fbb"}, {"c": "far a", "d": "far a far b"}, {"c": "baa a", "d": "baa bbb"}, {"c": "bar a", "d": "bar a bar b"}, ]
def test_filter_contains_not_in___lhs_is_int_rhs_is_list_of_lookups( runner_class, ): input_data = [ { "a": 1, "b": 2 }, { "a": 3, "b": 4 }, { "a": 5, "b": 6 }, { "a": 7, "b": 8 }, ] mapping = make_simple_mapping({ "c": [ TransformationEntry( transformation="a * 2", when="1 is not in [a, b]", ) ], "d": [ TransformationEntry( transformation="b + 3", when="6 is not in [a, b]", ) ], }) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": NotSet, "d": 5 }, { "c": 6, "d": 7 }, { "c": 10, "d": NotSet }, { "c": 14, "d": 11 }, ]
def get_transformations(self): return [ { "a": [TransformationEntry(transformation="a")], "b": [TransformationEntry(transformation="b")], "c": [TransformationEntry(transformation="a * 2")], "d": [TransformationEntry(transformation="b + 3")], }, ]
def test_filter_contains_or(runner_class): input_data = [ { "a": 1, "b": 2 }, { "a": 3, "b": 4 }, { "a": 5, "b": 6 }, { "a": 7, "b": 8 }, ] mapping = make_simple_mapping({ "c": [ TransformationEntry( transformation="a * 2", when="a is 1 or a is 5", ) ], "d": [ TransformationEntry( transformation="b + 3", when="b is 2 or b is 6", ) ], }) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": 2, "d": 5 }, { "c": 10, "d": 9 }, ]
def test_filter_contains_in___lhs_is_lookup_rhs_is_list_of_ints(runner_class): input_data = [ { "a": 1, "b": 2 }, { "a": 3, "b": 4 }, { "a": 5, "b": 6 }, { "a": 7, "b": 8 }, ] mapping = make_simple_mapping({ "c": [TransformationEntry( transformation="a * 2", when="a is in [1, 5]", )], "d": [TransformationEntry( transformation="b + 3", when="b is in [2, 6]", )], }) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": 2, "d": 5 }, { "c": 10, "d": 9 }, ]
def test_transform_contains_replace_with_multiple_pairs(runner_class): input_data = [ {"a": "foo a", "b": "foo b"}, {"a": "far a", "b": "far b"}, {"a": "boo a", "b": "boo b"}, {"a": "bar a", "b": "bar b"}, ] mapping = make_simple_mapping( { "c": [ TransformationEntry( transformation=""" replace( a + ' ' + b, 'foo', 'faa', 'boo', 'bam' ) """, ) ], } ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"c": "faa a faa b"}, {"c": "far a far b"}, {"c": "bam a bam b"}, {"c": "bar a bar b"}, ]
def test_transform_contains_join_of_lookups_str_and_non_str(runner_class): input_data = [ {"a": "foo a", "b": 1}, {"a": "far a", "b": 2}, {"a": "boo a", "b": 3}, {"a": "bar a", "b": 4}, ] mapping = make_simple_mapping( { "c": [ TransformationEntry( transformation="join(', ', 'bar', a, 5, b, 0)", ) ], }, types={"b": ColumnConversion(type="int")}, ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"c": "bar, foo a, 5, 1, 0"}, {"c": "bar, far a, 5, 2, 0"}, {"c": "bar, boo a, 5, 3, 0"}, {"c": "bar, bar a, 5, 4, 0"}, ]
def test_column_is_specified_as_string___values_are_changed_bad_are_excluded( runner_class, ): input_data = [ {"a": "1"}, {"a": 3.1}, {"a": None}, {"a": "NULL"}, {"a": "foo"}, ] mapping = make_simple_mapping( {"b": [TransformationEntry(transformation="a")]}, types={ "a": ColumnConversion(type="string", null_values=[None, "NULL"]), }, ) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(Config()).run(extractor, mapping, loader) assert list(loader.data) == [ {"b": "1"}, {"b": "3.1"}, {"b": None}, {"b": None}, {"b": "foo"}, ]
def test_filter_contains_fallback(runner_class): input_data = [ { "a": 1 }, { "a": 3 }, { "a": 5 }, { "a": 7 }, ] mapping = make_simple_mapping({ "c": [ TransformationEntry( transformation="a * 2", when="a gt 3", ), TransformationEntry(transformation="a * -1", ), ], }) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": -1 }, { "c": -3 }, { "c": 10 }, { "c": 14 }, ]
def test_filter_contains_all_is_in(runner_class): input_data = [ { "a": 1, "b": 2 }, { "a": 3, "b": 4 }, { "a": 5, "b": 6 }, { "a": 7, "b": 8 }, ] mapping = make_simple_mapping({ "c": [ TransformationEntry( transformation="a * 2", when="all [a, b] is not in [1, 2, 5, 6]", ) ], }) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": 6 }, { "c": 14 }, ]
def test_transform_contains_join_of_single_element(runner_class): input_data = [ { "a": "foo a", "b": 1 }, { "a": "far a", "b": 2 }, { "a": "boo a", "b": 3 }, { "a": "bar a", "b": 4 }, ] mapping = make_simple_mapping( {"c": [TransformationEntry(transformation="join(', ', a)", )]}) extractor = FakeConnector(data=input_data) loader = FakeConnector() runner_class(fake_transformation_config()).run(extractor, mapping, loader) assert list(loader.data) == [ { "c": "foo a" }, { "c": "far a" }, { "c": "boo a" }, { "c": "bar a" }, ]