def test_filter_contains_not___value_is_lookup(runner_class):
input_data = [
{
"a": 1,
"b": 2,
"c": 1
},
{
"a": 3,
"b": 4,
"c": 4
},
{
"a": 5,
"b": 6,
"c": 5
},
{
"a": 7,
"b": 8,
"c": 8
},
]
mapping = make_simple_mapping({
"c":
[TransformationEntry(
transformation="a * 2",
when="not (a is 1)",
)],
"d":
[TransformationEntry(
transformation="b + 3",
when="not (b is 6)",
)],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": NotSet,
"d": 5
},
{
"c": 6,
"d": 7
},
{
"c": 10,
"d": NotSet
},
{
"c": 14,
"d": 11
},
]
def test_when_contains_search_on_non_lookup(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="a", when="search('a foo b', 'foo')",
)
],
"d": [
TransformationEntry(
transformation="a + ' ' + b",
when=r"search('foo a bar', re'oo.\w')",
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "foo a", "d": "foo a foo b"},
{"c": "far a", "d": "far a far b"},
{"c": "boo a", "d": "boo a boo b"},
{"c": "bar a", "d": "bar a bar b"},
]
def test_transform_contains_replace_on_non_lookup(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="replace('foo', 'oo', 'aa')",
)
],
"d": [
TransformationEntry(
transformation=(r"replace('foo, bee', re'o{2}', 'aa')"),
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "faa", "d": "faa, bee"},
{"c": "faa", "d": "faa, bee"},
{"c": "faa", "d": "faa, bee"},
{"c": "faa", "d": "faa, bee"},
]
def test_when_contains_search(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="a", when="search(a, 'far')",
)
],
"d": [
TransformationEntry(
transformation="a + ' ' + b",
when=r"search(a + ' ' + b, re'a\s\w+oo')",
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": NotSet, "d": "foo a foo b"},
{"c": "far a", "d": NotSet},
{"c": NotSet, "d": "boo a boo b"},
]
def test_transform_contains_replace(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(transformation="replace(a, 'oo', 'aa')",)
],
"d": [
TransformationEntry(
transformation=(
r"replace(a + ' ' + b, re'oo (.)', '\1\1')"
),
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "faa a", "d": "faa fbb"},
{"c": "far a", "d": "far a far b"},
{"c": "baa a", "d": "baa bbb"},
{"c": "bar a", "d": "bar a bar b"},
]
def test_filter_contains_not_in___lhs_is_int_rhs_is_list_of_lookups(
runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c": [
TransformationEntry(
transformation="a * 2",
when="1 is not in [a, b]",
)
],
"d": [
TransformationEntry(
transformation="b + 3",
when="6 is not in [a, b]",
)
],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": NotSet,
"d": 5
},
{
"c": 6,
"d": 7
},
{
"c": 10,
"d": NotSet
},
{
"c": 14,
"d": 11
},
]
def get_transformations(self):
return [
{
"a": [TransformationEntry(transformation="a")],
"b": [TransformationEntry(transformation="b")],
"c": [TransformationEntry(transformation="a * 2")],
"d": [TransformationEntry(transformation="b + 3")],
},
]
def test_filter_contains_or(runner_class):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c": [
TransformationEntry(
transformation="a * 2",
when="a is 1 or a is 5",
)
],
"d": [
TransformationEntry(
transformation="b + 3",
when="b is 2 or b is 6",
)
],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": 2,
"d": 5
},
{
"c": 10,
"d": 9
},
]
def test_filter_contains_in___lhs_is_lookup_rhs_is_list_of_ints(runner_class):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c":
[TransformationEntry(
transformation="a * 2",
when="a is in [1, 5]",
)],
"d":
[TransformationEntry(
transformation="b + 3",
when="b is in [2, 6]",
)],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": 2,
"d": 5
},
{
"c": 10,
"d": 9
},
]
def test_transform_contains_replace_with_multiple_pairs(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="""
replace(
a + ' ' + b,
'foo', 'faa',
'boo', 'bam'
)
""",
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "faa a faa b"},
{"c": "far a far b"},
{"c": "bam a bam b"},
{"c": "bar a bar b"},
]
def test_transform_contains_join_of_lookups_str_and_non_str(runner_class):
input_data = [
{"a": "foo a", "b": 1},
{"a": "far a", "b": 2},
{"a": "boo a", "b": 3},
{"a": "bar a", "b": 4},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="join(', ', 'bar', a, 5, b, 0)",
)
],
},
types={"b": ColumnConversion(type="int")},
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "bar, foo a, 5, 1, 0"},
{"c": "bar, far a, 5, 2, 0"},
{"c": "bar, boo a, 5, 3, 0"},
{"c": "bar, bar a, 5, 4, 0"},
]
def test_column_is_specified_as_string___values_are_changed_bad_are_excluded(
runner_class,
):
input_data = [
{"a": "1"},
{"a": 3.1},
{"a": None},
{"a": "NULL"},
{"a": "foo"},
]
mapping = make_simple_mapping(
{"b": [TransformationEntry(transformation="a")]},
types={
"a": ColumnConversion(type="string", null_values=[None, "NULL"]),
},
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"b": "1"},
{"b": "3.1"},
{"b": None},
{"b": None},
{"b": "foo"},
]
def test_filter_contains_fallback(runner_class):
input_data = [
{
"a": 1
},
{
"a": 3
},
{
"a": 5
},
{
"a": 7
},
]
mapping = make_simple_mapping({
"c": [
TransformationEntry(
transformation="a * 2",
when="a gt 3",
),
TransformationEntry(transformation="a * -1", ),
],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": -1
},
{
"c": -3
},
{
"c": 10
},
{
"c": 14
},
]
def test_filter_contains_all_is_in(runner_class):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c": [
TransformationEntry(
transformation="a * 2",
when="all [a, b] is not in [1, 2, 5, 6]",
)
],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": 6
},
{
"c": 14
},
]
def test_transform_contains_join_of_single_element(runner_class):
input_data = [
{
"a": "foo a",
"b": 1
},
{
"a": "far a",
"b": 2
},
{
"a": "boo a",
"b": 3
},
{
"a": "bar a",
"b": 4
},
]
mapping = make_simple_mapping(
{"c": [TransformationEntry(transformation="join(', ', a)", )]})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": "foo a"
},
{
"c": "far a"
},
{
"c": "boo a"
},
{
"c": "bar a"
},
]
