def test_component_class_paths_default___default_component_classes_are_used():
runner = Mock()
extractor = Mock()
loader = Mock()
mapping = Mock()
with patch(
"converter.runner.PandasRunner",
return_value=runner,
) as runner_ctor_mock, patch(
"converter.connector.CsvConnector",
side_effect=[extractor, loader],
) as connector_ctor_mock, patch(
"converter.mapping.FileMapping",
return_value=mapping,
) as mapping_ctor_mock:
config = Config(
overrides={
"parallel": False,
"transformations": {
"ACC": {
"input_format": {
"name": "A",
"version": "1",
},
"output_format": {
"name": "B",
"version": "1",
},
"runner": {"options": {"first": "Some Runner Param"}},
"extractor": {
"options": {"first": "Some Extractor Param"}
},
"loader": {"options": {"first": "Some Loader Param"}},
"mapping": {
"options": {"first": "Some Mapping Param"}
},
}
},
}
)
controller = Controller(config)
controller.run()
transformer_config = config.get_transformation_configs()[0]
connector_ctor_mock.assert_any_call(
transformer_config, first="Some Extractor Param"
)
connector_ctor_mock.assert_any_call(
transformer_config, first="Some Loader Param"
)
mapping_ctor_mock.assert_called_once_with(
transformer_config, "acc", first="Some Mapping Param"
)
runner_ctor_mock.assert_called_once_with(
transformer_config, first="Some Runner Param"
)
runner.run.assert_called_once_with(extractor, mapping, loader)
def test_config_has_a_mixture_sources___normalised_sources_are_merged():
with config_file({
"foo": {
"BAR": "File Bar",
"boo": "File Boo",
"far": "File Far",
"dar": "File Dar",
}
}) as p:
config = Config(
config_path=p,
env={
"converter_foo_BAR": "Env Bar",
"converter_foo_boo": "Env Boo",
"converter_foo_far": "Env Far",
},
argv={
"foo.BAR": "Argv Bar",
"foo.boo": "Argv Boo"
},
overrides={"FOO": {
"bar": "Ovr Bar"
}},
)
assert config.get("foo.bar") == "Ovr Bar"
assert config.get("foo.boo") == "Argv Boo"
assert config.get("foo.far") == "Env Far"
assert config.get("foo.dar") == "File Dar"
def config(self):
config = Config(overrides=Config.merge_config_sources(
self._loaded_config,
self._default_working_config.config,
self._working_config.config,
), )
config.path = self._loaded_config.path
return config
async def test_base_async_transform_raises():
with pytest.raises(NotImplementedError):
[
row async for row in BaseAsyncRunner(Config()).transform(
BaseConnector(Config()),
BaseMapping(Config(), input_format="A", output_format="B"),
)
]
def test_config_has_list_in_env_options___normalised_from_env_are_used():
config = Config(
env={
"CONVERTER_FOO_BAR": '["BAz", "Buzz"]',
"converter_Foo_boo": "Fizz",
"ingored_boo_far": "ignored",
})
assert config.get("foo.bar") == ["BAz", "Buzz"]
assert config.get("foo.boo") == "Fizz"
def test_value_not_in_template_present_in_transformation____result_is_false():
conf = Config(
overrides={
"template_transformation": {
"b": "boo"
},
"transformations": {
"acc": {
"a": "bar"
}
},
})
assert not conf.uses_template_value("transformations.acc.a")
def test_value_in_template_matching_transformation____result_is_true():
conf = Config(
overrides={
"template_transformation": {
"a": "foo"
},
"transformations": {
"acc": {
"a": "foo"
}
},
})
assert conf.uses_template_value("transformations.acc.a")
def test_value_is_present__value_is_removed(config_path):
config = Config()
config.set(config_path, "foo")
config.delete(config_path)
with pytest.raises(KeyError):
config.get(config_path)
def test_transform_contains_replace(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(transformation="replace(a, 'oo', 'aa')",)
],
"d": [
TransformationEntry(
transformation=(
r"replace(a + ' ' + b, re'oo (.)', '\1\1')"
),
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "faa a", "d": "faa fbb"},
{"c": "far a", "d": "far a far b"},
{"c": "baa a", "d": "baa bbb"},
{"c": "bar a", "d": "bar a bar b"},
]
async def test_aload_is_not_implemented():
async def async_iter(data):
async for row in data:
yield row
with pytest.raises(NotImplementedError):
await BaseConnector(Config()).aload(async_iter([]))
def test_transform_contains_replace_with_multiple_pairs(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="""
replace(
a + ' ' + b,
'foo', 'faa',
'boo', 'bam'
)
""",
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "faa a faa b"},
{"c": "far a far b"},
{"c": "bam a bam b"},
{"c": "bar a bar b"},
]
def test_transform_contains_replace_on_non_lookup(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="replace('foo', 'oo', 'aa')",
)
],
"d": [
TransformationEntry(
transformation=(r"replace('foo, bee', re'o{2}', 'aa')"),
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "faa", "d": "faa, bee"},
{"c": "faa", "d": "faa, bee"},
{"c": "faa", "d": "faa, bee"},
{"c": "faa", "d": "faa, bee"},
]
def test_transform_contains_join_of_lookups_str_and_non_str(runner_class):
input_data = [
{"a": "foo a", "b": 1},
{"a": "far a", "b": 2},
{"a": "boo a", "b": 3},
{"a": "bar a", "b": 4},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="join(', ', 'bar', a, 5, b, 0)",
)
],
},
types={"b": ColumnConversion(type="int")},
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "bar, foo a, 5, 1, 0"},
{"c": "bar, far a, 5, 2, 0"},
{"c": "bar, boo a, 5, 3, 0"},
{"c": "bar, bar a, 5, 4, 0"},
]
def test_when_contains_search_on_non_lookup(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="a", when="search('a foo b', 'foo')",
)
],
"d": [
TransformationEntry(
transformation="a + ' ' + b",
when=r"search('foo a bar', re'oo.\w')",
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": "foo a", "d": "foo a foo b"},
{"c": "far a", "d": "far a far b"},
{"c": "boo a", "d": "boo a boo b"},
{"c": "bar a", "d": "bar a bar b"},
]
def test_when_contains_search(runner_class):
input_data = [
{"a": "foo a", "b": "foo b"},
{"a": "far a", "b": "far b"},
{"a": "boo a", "b": "boo b"},
{"a": "bar a", "b": "bar b"},
]
mapping = make_simple_mapping(
{
"c": [
TransformationEntry(
transformation="a", when="search(a, 'far')",
)
],
"d": [
TransformationEntry(
transformation="a + ' ' + b",
when=r"search(a + ' ' + b, re'a\s\w+oo')",
)
],
}
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"c": NotSet, "d": "foo a foo b"},
{"c": "far a", "d": NotSet},
{"c": NotSet, "d": "boo a boo b"},
]
def test_filter_contains_not___value_is_lookup(runner_class):
input_data = [
{
"a": 1,
"b": 2,
"c": 1
},
{
"a": 3,
"b": 4,
"c": 4
},
{
"a": 5,
"b": 6,
"c": 5
},
{
"a": 7,
"b": 8,
"c": 8
},
]
mapping = make_simple_mapping({
"c":
[TransformationEntry(
transformation="a * 2",
when="not (a is 1)",
)],
"d":
[TransformationEntry(
transformation="b + 3",
when="not (b is 6)",
)],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": NotSet,
"d": 5
},
{
"c": 6,
"d": 7
},
{
"c": 10,
"d": NotSet
},
{
"c": 14,
"d": 11
},
]
def cli(ctx, config, verbose, no_color, option):
"""
Initialises the cli grouping with default options.
"""
ctx.ensure_object(dict)
init_logging(verbose, no_color, config)
options = dict(option)
ctx.obj["config"] = Config(
config_path=config,
argv={k: yaml.load(v, yaml.SafeLoader)
for k, v in options.items()},
env=os.environ,
)
if ctx.invoked_subcommand is None:
app = QApplication(sys.argv)
widget = MainWindow(ctx.obj["config"],
lambda p: init_logging(verbose, no_color, p))
widget.show()
sys.exit(app.exec_())
def test_column_is_specified_as_string___values_are_changed_bad_are_excluded(
runner_class,
):
input_data = [
{"a": "1"},
{"a": 3.1},
{"a": None},
{"a": "NULL"},
{"a": "foo"},
]
mapping = make_simple_mapping(
{"b": [TransformationEntry(transformation="a")]},
types={
"a": ColumnConversion(type="string", null_values=[None, "NULL"]),
},
)
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{"b": "1"},
{"b": "3.1"},
{"b": None},
{"b": None},
{"b": "foo"},
]
def test_filter_contains_not_in___lhs_is_int_rhs_is_list_of_lookups(
runner_class, ):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c": [
TransformationEntry(
transformation="a * 2",
when="1 is not in [a, b]",
)
],
"d": [
TransformationEntry(
transformation="b + 3",
when="6 is not in [a, b]",
)
],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": NotSet,
"d": 5
},
{
"c": 6,
"d": 7
},
{
"c": 10,
"d": NotSet
},
{
"c": 14,
"d": 11
},
]
def test_tpl_present___has_template_is_true():
assert not Config(overrides={
"transformation_template": {
"acc": {
"foo": "bar"
}
}
}).has_acc
def test_loc_present___has_loc_is_true():
assert Config(overrides={
"transformations": {
"loc": {
"foo": "bar"
}
}
}).has_loc
def test_filename_provided___original_file_is_left_intact():
with config_file({}) as p:
new_conf_path = os.path.join(os.path.dirname(p), "new-conf.yml")
orig_config = Config(config_path=p)
updated_config = Config(config_path=p)
updated_config.set("foo", "bar")
updated_config.save(new_filename=new_conf_path)
reloaded = Config(config_path=p)
assert reloaded == orig_config
assert updated_config == Config(config_path=new_conf_path)
def test_value_is_not_present__config_is_unchanged(config_path):
config = Config(overrides={"bar": "baz"})
config.delete(config_path)
assert config.get("bar") == "baz"
with pytest.raises(KeyError):
config.get(config_path)
def __init__(self, config, update_log_paths):
super().__init__()
# initialise the config
self._loaded_config = config
self._working_config = Config()
self._default_working_config = Config()
# setup the top menu
self._create_actions()
self._create_menu_bar()
self.minimumWidth = 750
# setup tabs
self.tabs = QTabWidget()
self.tabs.tabsClosable = True
self.tabs.tabCloseRequested.connect(self.on_close_tab)
self.metadata_tab = MetadataTab(self)
meta_scroll_wrapper = QScrollArea()
meta_scroll_wrapper.setWidget(self.metadata_tab)
self.tabs.addTab(meta_scroll_wrapper, "Metadata")
self.tabs.tabBar().setTabButton(0, QTabBar.RightSide, None)
self.run_tab = RunTab(self)
self.tabs.addTab(self.run_tab, "Run")
self.tabs.tabBar().setTabButton(1, QTabBar.RightSide, None)
# add create tab button
self.tab_button = AddTabButton(self)
self.tabs.setCornerWidget(self.tab_button)
self.config_tabs = {}
self.initialise_config_tabs(self.config)
self.config_changed.connect(self.initialise_config_tabs)
self.setCentralWidget(self.tabs)
self.running_changed.connect(
lambda b: self.menuBar().setEnabled(not b))
self.update_log_paths = update_log_paths
def test_filename__not_provided___original_file_updated():
with config_file({}) as p:
updated_config = Config(config_path=p)
updated_config.set("foo", "bar")
updated_config.save()
reloaded = Config(config_path=p)
assert updated_config == reloaded
def test_component_class_paths_default___default_component_classes_are_used():
runner = Mock()
extractor = Mock()
loader = Mock()
mapping = Mock()
with patch(
"converter.runner.PandasRunner",
return_value=runner,
) as runner_ctor_mock, patch(
"converter.connector.CsvConnector",
side_effect=[extractor, loader],
) as connector_ctor_mock, patch(
"converter.mapping.FileMapping",
return_value=mapping,
) as mapping_ctor_mock:
config = Config(
overrides={
"runner": {
"options": {
"first": "Some Runner Param"
}
},
"extractor": {
"options": {
"first": "Some Extractor Param"
}
},
"loader": {
"options": {
"first": "Some Loader Param"
}
},
"mapping": {
"options": {
"first": "Some Mapping Param"
}
},
})
controller = Controller(config)
controller.run()
connector_ctor_mock.assert_any_call(config,
first="Some Extractor Param")
connector_ctor_mock.assert_any_call(config, first="Some Loader Param")
mapping_ctor_mock.assert_called_once_with(config,
first="Some Mapping Param")
runner_ctor_mock.assert_called_once_with(config,
first="Some Runner Param")
runner.run.assert_called_once_with(extractor, mapping, loader)
def __init__(
self,
input_format,
output_format,
specs: List[MappingSpec],
config=None,
):
super().__init__(
config or Config(),
input_format=input_format,
output_format=output_format,
)
self.specs = specs
def test_filter_contains_or(runner_class):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c": [
TransformationEntry(
transformation="a * 2",
when="a is 1 or a is 5",
)
],
"d": [
TransformationEntry(
transformation="b + 3",
when="b is 2 or b is 6",
)
],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": 2,
"d": 5
},
{
"c": 10,
"d": 9
},
]
def reset_changes(self, file_path):
self._loaded_config = Config(
config_path=file_path,
overrides=self._loaded_config.overrides,
env=self._loaded_config.env,
argv=self._loaded_config.argv,
)
self._working_config = Config()
self._default_working_config = Config()
self.config_changed.emit(self.config)
def test_filter_contains_in___lhs_is_lookup_rhs_is_list_of_ints(runner_class):
input_data = [
{
"a": 1,
"b": 2
},
{
"a": 3,
"b": 4
},
{
"a": 5,
"b": 6
},
{
"a": 7,
"b": 8
},
]
mapping = make_simple_mapping({
"c":
[TransformationEntry(
transformation="a * 2",
when="a is in [1, 5]",
)],
"d":
[TransformationEntry(
transformation="b + 3",
when="b is in [2, 6]",
)],
})
extractor = FakeConnector(data=input_data)
loader = FakeConnector()
runner_class(Config()).run(extractor, mapping, loader)
assert list(loader.data) == [
{
"c": 2,
"d": 5
},
{
"c": 10,
"d": 9
},
]