def test_column_is_specified_as_string___values_are_changed_bad_are_excluded(
runner_class,
):
input_data = [
{"a": "1"},
{"a": 3.1},
{"a": None},
{"a": "NULL"},
{"a": "foo"},
]
mapping = make_simple_mapping(
{"b": [TransformationEntry(transformation="a")]},
types={
"a": ColumnConversion(type="string", null_values=[None, "NULL"]),
},
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"b": "1"},
{"b": "3.1"},
{"b": None},
{"b": None},
{"b": "foo"},
]
def test_row_is_value___value_is_set_on_all_columns(runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
with TemporaryDirectory() as search:
write_yaml(
os.path.join(search, "A-B.yml"),
{
"input_format": "A",
"output_format": "B",
"forward": {
"transform": {
"c": [{
"transformation": "a * 2"
}],
"d": [{
"transformation": "b + 3"
}],
"e": [{
"transformation": "'foo'"
}],
}
},
"reverse": {
"transform": {
"a": [{
"transformation": "c / 2"
}],
"b": [{
"transformation": "d - 3"
}],
}
},
},
)
# run forward
forward_mapping = FileMapping(
Config(),
input_format="A",
output_format="B",
standard_search_path=search,
search_working_dir=False,
)
forward_extractor = FakeConnector(data=input_data)
forward_loader = FakeConnector()
runner_class(Config()).run(forward_extractor, forward_mapping,
forward_loader)
assert list(forward_loader.data) == [
{
"c": 2,
"d": 5,
"e": "foo"
},
{
"c": 6,
"d": 7,
"e": "foo"
},
{
"c": 10,
"d": 9,
"e": "foo"
},
{
"c": 14,
"d": 11,
"e": "foo"
},
]
# reverse runner
reverse_mapping = FileMapping(
Config(),
input_format="B",
output_format="A",
standard_search_path=search,
search_working_dir=False,
)
reverse_extractor = forward_loader
reverse_loader = FakeConnector()
runner_class(Config()).run(reverse_extractor, reverse_mapping,
reverse_loader)
assert list(reverse_loader.data) == input_data
def test_multiple_mapping_steps___forward_and_reverse_gets_to_the_input(
runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
with TemporaryDirectory() as search:
write_yaml(
os.path.join(search, "A-B.yml"),
{
"input_format": "A",
"output_format": "B",
"forward": {
"transform": {
"c": [{
"transformation": "a * 2"
}],
"d": [{
"transformation": "b + 3"
}],
}
},
"reverse": {
"transform": {
"a": [{
"transformation": "c / 2"
}],
"b": [{
"transformation": "d - 3"
}],
}
},
},
)
write_yaml(
os.path.join(search, "B-C.yml"),
{
"input_format": "B",
"output_format": "C",
"forward": {
"transform": {
"e": [{
"transformation": "c * 3"
}],
"f": [{
"transformation": "d + 4"
}],
}
},
"reverse": {
"transform": {
"c": [{
"transformation": "e / 3"
}],
"d": [{
"transformation": "f - 4"
}],
}
},
},
)
# run forward
forward_mapping = FileMapping(
Config(),
input_format="A",
output_format="C",
standard_search_path=search,
search_working_dir=False,
)
forward_extractor = FakeConnector(data=input_data)
forward_loader = FakeConnector()
runner_class(Config()).run(forward_extractor, forward_mapping,
forward_loader)
assert list(forward_loader.data) == [
{
"e": 6,
"f": 9
},
{
"e": 18,
"f": 11
},
{
"e": 30,
"f": 13
},
{
"e": 42,
"f": 15
},
]
# reverse runner
reverse_mapping = FileMapping(
Config(),
input_format="C",
output_format="A",
standard_search_path=search,
search_working_dir=False,
)
reverse_extractor = forward_loader
reverse_loader = FakeConnector()
runner_class(Config()).run(reverse_extractor, reverse_mapping,
reverse_loader)
assert list(reverse_loader.data) == input_data
def test_multiple_transforms_could_apply___first_is_applied(runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
with TemporaryDirectory() as search:
write_yaml(
os.path.join(search, "A-B.yml"),
{
"file_type": "ACC",
"input_format": {
"name": "A",
"version": "1"
},
"output_format": {
"name": "B",
"version": "1"
},
"forward": {
"transform": {
"c": [
{
"transformation": "a * 2"
},
{
"transformation": "a * 4"
},
],
"d": [{
"transformation": "b + 3"
}],
}
},
"reverse": {
"transform": {
"a": [{
"transformation": "c / 2"
}],
"b": [{
"transformation": "d - 3"
}],
}
},
},
)
# run forward
forward_mapping = FileMapping(
fake_transformation_config(),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
forward_extractor = FakeConnector(data=input_data)
forward_loader = FakeConnector()
runner_class(fake_transformation_config()).run(forward_extractor,
forward_mapping,
forward_loader)
assert list(forward_loader.data) == [
{
"c": 2,
"d": 5
},
{
"c": 6,
"d": 7
},
{
"c": 10,
"d": 9
},
{
"c": 14,
"d": 11
},
]
# reverse runner
reverse_mapping = FileMapping(
fake_transformation_config({
"transformations": {
"ACC": {
"input_format": {
"name": "B",
"version": "1",
},
"output_format": {
"name": "A",
"version": "1",
},
}
}
}),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
reverse_extractor = forward_loader
reverse_loader = FakeConnector()
runner_class(fake_transformation_config()).run(reverse_extractor,
reverse_mapping,
reverse_loader)
assert list(reverse_loader.data) == input_data