예제 #1
0
def test_filter_contains_not___value_is_lookup(runner_class):
    input_data = [
        {
            "a": 1,
            "b": 2,
            "c": 1
        },
        {
            "a": 3,
            "b": 4,
            "c": 4
        },
        {
            "a": 5,
            "b": 6,
            "c": 5
        },
        {
            "a": 7,
            "b": 8,
            "c": 8
        },
    ]

    mapping = make_simple_mapping({
        "c":
        [TransformationEntry(
            transformation="a * 2",
            when="not (a is 1)",
        )],
        "d":
        [TransformationEntry(
            transformation="b + 3",
            when="not (b is 6)",
        )],
    })

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": NotSet,
            "d": 5
        },
        {
            "c": 6,
            "d": 7
        },
        {
            "c": 10,
            "d": NotSet
        },
        {
            "c": 14,
            "d": 11
        },
    ]
def test_when_contains_search_on_non_lookup(runner_class):
    input_data = [
        {"a": "foo a", "b": "foo b"},
        {"a": "far a", "b": "far b"},
        {"a": "boo a", "b": "boo b"},
        {"a": "bar a", "b": "bar b"},
    ]

    mapping = make_simple_mapping(
        {
            "c": [
                TransformationEntry(
                    transformation="a", when="search('a foo b', 'foo')",
                )
            ],
            "d": [
                TransformationEntry(
                    transformation="a + ' ' + b",
                    when=r"search('foo a bar', re'oo.\w')",
                )
            ],
        }
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"c": "foo a", "d": "foo a foo b"},
        {"c": "far a", "d": "far a far b"},
        {"c": "boo a", "d": "boo a boo b"},
        {"c": "bar a", "d": "bar a bar b"},
    ]
def test_transform_contains_replace_on_non_lookup(runner_class):
    input_data = [
        {"a": "foo a", "b": "foo b"},
        {"a": "far a", "b": "far b"},
        {"a": "boo a", "b": "boo b"},
        {"a": "bar a", "b": "bar b"},
    ]

    mapping = make_simple_mapping(
        {
            "c": [
                TransformationEntry(
                    transformation="replace('foo', 'oo', 'aa')",
                )
            ],
            "d": [
                TransformationEntry(
                    transformation=(r"replace('foo, bee', re'o{2}', 'aa')"),
                )
            ],
        }
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"c": "faa", "d": "faa, bee"},
        {"c": "faa", "d": "faa, bee"},
        {"c": "faa", "d": "faa, bee"},
        {"c": "faa", "d": "faa, bee"},
    ]
def test_when_contains_search(runner_class):
    input_data = [
        {"a": "foo a", "b": "foo b"},
        {"a": "far a", "b": "far b"},
        {"a": "boo a", "b": "boo b"},
        {"a": "bar a", "b": "bar b"},
    ]

    mapping = make_simple_mapping(
        {
            "c": [
                TransformationEntry(
                    transformation="a", when="search(a, 'far')",
                )
            ],
            "d": [
                TransformationEntry(
                    transformation="a + ' ' + b",
                    when=r"search(a + ' ' + b, re'a\s\w+oo')",
                )
            ],
        }
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"c": NotSet, "d": "foo a foo b"},
        {"c": "far a", "d": NotSet},
        {"c": NotSet, "d": "boo a boo b"},
    ]
def test_transform_contains_replace(runner_class):
    input_data = [
        {"a": "foo a", "b": "foo b"},
        {"a": "far a", "b": "far b"},
        {"a": "boo a", "b": "boo b"},
        {"a": "bar a", "b": "bar b"},
    ]

    mapping = make_simple_mapping(
        {
            "c": [
                TransformationEntry(transformation="replace(a, 'oo', 'aa')",)
            ],
            "d": [
                TransformationEntry(
                    transformation=(
                        r"replace(a + ' ' + b, re'oo (.)', '\1\1')"
                    ),
                )
            ],
        }
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"c": "faa a", "d": "faa fbb"},
        {"c": "far a", "d": "far a far b"},
        {"c": "baa a", "d": "baa bbb"},
        {"c": "bar a", "d": "bar a bar b"},
    ]
예제 #6
0
def test_filter_contains_not_in___lhs_is_int_rhs_is_list_of_lookups(
    runner_class, ):
    input_data = [
        {
            "a": 1,
            "b": 2
        },
        {
            "a": 3,
            "b": 4
        },
        {
            "a": 5,
            "b": 6
        },
        {
            "a": 7,
            "b": 8
        },
    ]

    mapping = make_simple_mapping({
        "c": [
            TransformationEntry(
                transformation="a * 2",
                when="1 is not in [a, b]",
            )
        ],
        "d": [
            TransformationEntry(
                transformation="b + 3",
                when="6 is not in [a, b]",
            )
        ],
    })

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": NotSet,
            "d": 5
        },
        {
            "c": 6,
            "d": 7
        },
        {
            "c": 10,
            "d": NotSet
        },
        {
            "c": 14,
            "d": 11
        },
    ]
예제 #7
0
 def get_transformations(self):
     return [
         {
             "a": [TransformationEntry(transformation="a")],
             "b": [TransformationEntry(transformation="b")],
             "c": [TransformationEntry(transformation="a * 2")],
             "d": [TransformationEntry(transformation="b + 3")],
         },
     ]
예제 #8
0
def test_filter_contains_or(runner_class):
    input_data = [
        {
            "a": 1,
            "b": 2
        },
        {
            "a": 3,
            "b": 4
        },
        {
            "a": 5,
            "b": 6
        },
        {
            "a": 7,
            "b": 8
        },
    ]

    mapping = make_simple_mapping({
        "c": [
            TransformationEntry(
                transformation="a * 2",
                when="a is 1 or a is 5",
            )
        ],
        "d": [
            TransformationEntry(
                transformation="b + 3",
                when="b is 2 or b is 6",
            )
        ],
    })

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": 2,
            "d": 5
        },
        {
            "c": 10,
            "d": 9
        },
    ]
예제 #9
0
def test_filter_contains_in___lhs_is_lookup_rhs_is_list_of_ints(runner_class):
    input_data = [
        {
            "a": 1,
            "b": 2
        },
        {
            "a": 3,
            "b": 4
        },
        {
            "a": 5,
            "b": 6
        },
        {
            "a": 7,
            "b": 8
        },
    ]

    mapping = make_simple_mapping({
        "c":
        [TransformationEntry(
            transformation="a * 2",
            when="a is in [1, 5]",
        )],
        "d":
        [TransformationEntry(
            transformation="b + 3",
            when="b is in [2, 6]",
        )],
    })

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": 2,
            "d": 5
        },
        {
            "c": 10,
            "d": 9
        },
    ]
def test_transform_contains_replace_with_multiple_pairs(runner_class):
    input_data = [
        {"a": "foo a", "b": "foo b"},
        {"a": "far a", "b": "far b"},
        {"a": "boo a", "b": "boo b"},
        {"a": "bar a", "b": "bar b"},
    ]

    mapping = make_simple_mapping(
        {
            "c": [
                TransformationEntry(
                    transformation="""
                        replace(
                            a + ' ' + b,
                            'foo', 'faa',
                            'boo', 'bam'
                        )
                    """,
                )
            ],
        }
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"c": "faa a faa b"},
        {"c": "far a far b"},
        {"c": "bam a bam b"},
        {"c": "bar a bar b"},
    ]
def test_transform_contains_join_of_lookups_str_and_non_str(runner_class):
    input_data = [
        {"a": "foo a", "b": 1},
        {"a": "far a", "b": 2},
        {"a": "boo a", "b": 3},
        {"a": "bar a", "b": 4},
    ]

    mapping = make_simple_mapping(
        {
            "c": [
                TransformationEntry(
                    transformation="join(', ', 'bar', a, 5, b, 0)",
                )
            ],
        },
        types={"b": ColumnConversion(type="int")},
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"c": "bar, foo a, 5, 1, 0"},
        {"c": "bar, far a, 5, 2, 0"},
        {"c": "bar, boo a, 5, 3, 0"},
        {"c": "bar, bar a, 5, 4, 0"},
    ]
def test_column_is_specified_as_string___values_are_changed_bad_are_excluded(
    runner_class,
):
    input_data = [
        {"a": "1"},
        {"a": 3.1},
        {"a": None},
        {"a": "NULL"},
        {"a": "foo"},
    ]

    mapping = make_simple_mapping(
        {"b": [TransformationEntry(transformation="a")]},
        types={
            "a": ColumnConversion(type="string", null_values=[None, "NULL"]),
        },
    )

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(Config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {"b": "1"},
        {"b": "3.1"},
        {"b": None},
        {"b": None},
        {"b": "foo"},
    ]
예제 #13
0
def test_filter_contains_fallback(runner_class):
    input_data = [
        {
            "a": 1
        },
        {
            "a": 3
        },
        {
            "a": 5
        },
        {
            "a": 7
        },
    ]

    mapping = make_simple_mapping({
        "c": [
            TransformationEntry(
                transformation="a * 2",
                when="a gt 3",
            ),
            TransformationEntry(transformation="a * -1", ),
        ],
    })

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": -1
        },
        {
            "c": -3
        },
        {
            "c": 10
        },
        {
            "c": 14
        },
    ]
예제 #14
0
def test_filter_contains_all_is_in(runner_class):
    input_data = [
        {
            "a": 1,
            "b": 2
        },
        {
            "a": 3,
            "b": 4
        },
        {
            "a": 5,
            "b": 6
        },
        {
            "a": 7,
            "b": 8
        },
    ]

    mapping = make_simple_mapping({
        "c": [
            TransformationEntry(
                transformation="a * 2",
                when="all [a, b] is not in [1, 2, 5, 6]",
            )
        ],
    })

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": 6
        },
        {
            "c": 14
        },
    ]
예제 #15
0
def test_transform_contains_join_of_single_element(runner_class):
    input_data = [
        {
            "a": "foo a",
            "b": 1
        },
        {
            "a": "far a",
            "b": 2
        },
        {
            "a": "boo a",
            "b": 3
        },
        {
            "a": "bar a",
            "b": 4
        },
    ]

    mapping = make_simple_mapping(
        {"c": [TransformationEntry(transformation="join(', ', a)", )]})

    extractor = FakeConnector(data=input_data)
    loader = FakeConnector()

    runner_class(fake_transformation_config()).run(extractor, mapping, loader)

    assert list(loader.data) == [
        {
            "c": "foo a"
        },
        {
            "c": "far a"
        },
        {
            "c": "boo a"
        },
        {
            "c": "bar a"
        },
    ]