def test_base_transform_raises():
with pytest.raises(NotImplementedError):
BaseRunner(fake_transformation_config()).transform(
BaseConnector(fake_transformation_config()),
BaseMapping(
fake_transformation_config(),
"ACC",
),
)
async def test_base_async_transform_raises():
with pytest.raises(NotImplementedError):
[
row async for row in BaseAsyncRunner(
fake_transformation_config()).transform(
BaseConnector(fake_transformation_config()),
BaseMapping(
fake_transformation_config(),
"ACC",
),
)
]
async def test_aload_is_not_implemented():
async def async_iter(data):
async for row in data:
yield row
with pytest.raises(NotImplementedError):
await BaseConnector(fake_transformation_config()).aload(async_iter([]))
def test_column_is_specified_as_string___values_are_changed_bad_are_excluded(
runner_class,
):
input_data = [
{"a": "1"},
{"a": 3.1},
{"a": None},
{"a": "NULL"},
{"a": "foo"},
]
mapping = make_simple_mapping(
{"b": [TransformationEntry(transformation="a")]},
types={
"a": ColumnConversion(type="string", null_values=[None, "NULL"]),
},
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(fake_transformation_config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"b": "1"},
{"b": "3.1"},
{"b": None},
{"b": None},
{"b": "foo"},
]
def test_data_is_passed_to_loader_no_header___data_is_written_to_file():
expected_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
]
with TemporaryDirectory() as p:
output_path = os.path.join(p, "result.csv")
CsvConnector(
fake_transformation_config(),
path=output_path,
write_header=False,
).load(expected_data)
with open(output_path, "r") as f:
assert (list(
csv.DictReader(
f,
quoting=csv.QUOTE_NONNUMERIC,
fieldnames=["a", "b"],
)) == expected_data)
def test_no_data_is_passed_to_loader___no_file_is_written():
with TemporaryDirectory() as p:
output_path = os.path.join(p, "result.csv")
CsvConnector(fake_transformation_config(), path=output_path).load([])
assert not os.path.exists(output_path)
def test_file_contains_data_with_heading___all_entries_are_loaded():
expected_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
]
with NamedTemporaryFile("w+", newline="") as f:
writer = csv.DictWriter(f,
fieldnames=["a", "b"],
quoting=csv.QUOTE_NONNUMERIC)
writer.writeheader()
writer.writerows(expected_data)
f.flush()
assert (list(
CsvConnector(fake_transformation_config(),
path=f.name).extract()) == expected_data)
def test_get_transformations_raises_a_not_implemented_error():
with pytest.raises(NotImplementedError):
mapping = BaseMapping(
fake_transformation_config(),
"ACC",
input_format=MappingFormat(name="A", version="1"),
output_format=MappingFormat(name="B", version="1"),
)
mapping.get_transformations()
def __init__(
self,
input_format,
output_format,
specs: List[MappingSpec],
config=None,
):
super().__init__(
config or fake_transformation_config(),
"ACC",
input_format=input_format,
output_format=output_format,
)
self.specs = specs
def __init__(self, data=None, **options):
super().__init__(fake_transformation_config(), **options)
self.data = data or []
def test_extract_is_not_implemented():
with pytest.raises(NotImplementedError):
BaseConnector(fake_transformation_config()).extract()
async def test_aextract_is_not_implemented():
with pytest.raises(NotImplementedError):
[
row async for row in BaseConnector(
fake_transformation_config()).aextract()
]
def test_row_is_value___value_is_set_on_all_columns(runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
with TemporaryDirectory() as search:
write_yaml(
os.path.join(search, "A-B.yml"),
{
"file_type": "ACC",
"input_format": {
"name": "A",
"version": "1"
},
"output_format": {
"name": "B",
"version": "1"
},
"forward": {
"transform": {
"c": [{
"transformation": "a * 2"
}],
"d": [{
"transformation": "b + 3"
}],
"e": [{
"transformation": "'foo'"
}],
}
},
"reverse": {
"transform": {
"a": [{
"transformation": "c / 2"
}],
"b": [{
"transformation": "d - 3"
}],
}
},
},
)
# run forward
forward_mapping = FileMapping(
fake_transformation_config(),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
forward_extractor = FakeConnector(data=input_data)
forward_loader = FakeConnector()
runner_class(fake_transformation_config()).run(forward_extractor,
forward_mapping,
forward_loader)
assert list(forward_loader.data) == [
{
"c": 2,
"d": 5,
"e": "foo"
},
{
"c": 6,
"d": 7,
"e": "foo"
},
{
"c": 10,
"d": 9,
"e": "foo"
},
{
"c": 14,
"d": 11,
"e": "foo"
},
]
# reverse runner
reverse_mapping = FileMapping(
fake_transformation_config({
"transformations": {
"ACC": {
"input_format": {
"name": "B",
"version": "1",
},
"output_format": {
"name": "A",
"version": "1",
},
}
}
}),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
reverse_extractor = forward_loader
reverse_loader = FakeConnector()
runner_class(fake_transformation_config()).run(reverse_extractor,
reverse_mapping,
reverse_loader)
assert list(reverse_loader.data) == input_data
def test_multiple_mapping_steps___forward_and_reverse_gets_to_the_input(
runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
with TemporaryDirectory() as search:
write_yaml(
os.path.join(search, "A-B.yml"),
{
"file_type": "ACC",
"input_format": {
"name": "A",
"version": "1"
},
"output_format": {
"name": "B",
"version": "1"
},
"forward": {
"transform": {
"c": [{
"transformation": "a * 2"
}],
"d": [{
"transformation": "b + 3"
}],
}
},
"reverse": {
"transform": {
"a": [{
"transformation": "c / 2"
}],
"b": [{
"transformation": "d - 3"
}],
}
},
},
)
write_yaml(
os.path.join(search, "B-C.yml"),
{
"file_type": "ACC",
"input_format": {
"name": "B",
"version": "1"
},
"output_format": {
"name": "C",
"version": "1"
},
"forward": {
"transform": {
"e": [{
"transformation": "c * 3"
}],
"f": [{
"transformation": "d + 4"
}],
}
},
"reverse": {
"transform": {
"c": [{
"transformation": "e / 3"
}],
"d": [{
"transformation": "f - 4"
}],
}
},
},
)
# run forward
forward_mapping = FileMapping(
fake_transformation_config({
"transformations": {
"ACC": {
"input_format": {
"name": "A",
"version": "1",
},
"output_format": {
"name": "C",
"version": "1",
},
}
}
}),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
forward_extractor = FakeConnector(data=input_data)
forward_loader = FakeConnector()
runner_class(fake_transformation_config()).run(forward_extractor,
forward_mapping,
forward_loader)
assert list(forward_loader.data) == [
{
"e": 6,
"f": 9
},
{
"e": 18,
"f": 11
},
{
"e": 30,
"f": 13
},
{
"e": 42,
"f": 15
},
]
# reverse runner
reverse_mapping = FileMapping(
fake_transformation_config({
"transformations": {
"ACC": {
"input_format": {
"name": "C",
"version": "1",
},
"output_format": {
"name": "A",
"version": "1",
},
}
}
}),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
reverse_extractor = forward_loader
reverse_loader = FakeConnector()
runner_class(fake_transformation_config()).run(reverse_extractor,
reverse_mapping,
reverse_loader)
assert list(reverse_loader.data) == input_data
def test_runner_handles_when_secondary_cases_are_false():
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
with TemporaryDirectory() as search:
write_yaml(
os.path.join(search, "A-B.yml"),
{
"file_type": "ACC",
"input_format": {
"name": "A",
"version": "1"
},
"output_format": {
"name": "B",
"version": "1"
},
"forward": {
"transform": {
"c": [
{
"transformation": "a * 2"
},
{
"transformation": "a * 3",
"when": "False"
},
],
"d": [{
"transformation": "b + 3"
}],
}
},
},
)
# run forward
forward_mapping = FileMapping(
fake_transformation_config(),
"ACC",
standard_search_path=search,
search_working_dir=False,
)
forward_extractor = FakeConnector(data=input_data)
forward_loader = FakeConnector()
PandasRunner(fake_transformation_config()).run(forward_extractor,
forward_mapping,
forward_loader)
assert list(forward_loader.data) == [
{
"c": 2,
"d": 5
},
{
"c": 6,
"d": 7
},
{
"c": 10,
"d": 9
},
{
"c": 14,
"d": 11
},
]
def test_file_does_not_exist___error_is_raised():
connector = CsvConnector(fake_transformation_config(), path="somepath.csv")
with pytest.raises(FileNotFoundError):
list(connector.extract())
