def test_filter_sequence4():
config = {
"metadata": {
"section_run": ["writer_config"]
},
"implementation_config": {
"reader_config": {
"read_data": {
"class": "CsvReader",
"filename": "test/tennis.csv",
"destinations": ["recipe_csv_writer"],
}
},
"writer_config": {
"recipe_csv_writer": {
"class": "CsvWriter",
"key": "test_data",
"dir": "cache",
"filename": "unittest_similar_recipes.csv",
}
},
},
}
configuration = Configuration(None,
is_dict_config=True,
dict_config=config)
runner = DagRunner(configuration)
sequence = runner.filter_sequence(["read_data", "recipe_csv_writer"])
assert sequence == ["recipe_csv_writer"]
def test_filter_sequence6():
# test of dependencies with a section
config = {
"metadata": {
"traverser": "DepthFirstTraverser"
},
"implementation_config": {
"reader_config": {
"read_data1": {
"class": "CsvReader",
"filename": "test/tennis.csv",
"destinations": ["csv_writer"],
},
"read_data2": {
"class": "CsvReader",
"filename": "test/tennis.csv",
"destinations": ["csv_writer"],
},
"read_data3": {
"class": "CsvReader",
"filename": "test/tennis.csv",
"destinations": ["read_data2"],
},
},
"writer_config": {
"csv_writer": {
"class": "CsvWriter",
"key": "test_data",
"dir": "cache",
"filename": "unittest_similar_recipes.csv",
}
},
},
}
configuration = Configuration(None,
is_dict_config=True,
dict_config=config)
runner = DagRunner(configuration)
s_in = ["read_data3", "read_data2", "read_data1", "csv_writer"]
s_out = runner.filter_sequence(s_in)
assert s_out == s_in
with pytest.raises(Exception) as e:
runner.filter_sequence(
["read_data1", "csv_writer", "read_data3", "read_data2"])
assert "Upstream path found, from read_data3 to csv_writer" in str(e.value)
def test_filter_sequence5():
config = {
"metadata": {
"section_run": ["writer_config", "reader_config"]
},
"implementation_config": {
"reader_config": {
"read_data": {
"class": "CsvReader",
"filename": "test/tennis.csv",
"destinations":
["recipe_csv_writer", "recipe_csv_writer2"],
}
},
"writer_config": {
"recipe_csv_writer": {
"class": "CsvWriter",
"key": "test_data",
"dir": "cache",
"filename": "unittest_similar_recipes.csv",
},
"recipe_csv_writer2": {
"class": "CsvWriter",
"key": "test_data",
"dir": "cache",
"filename": "unittest_similar_recipes.csv",
},
},
},
}
configuration = Configuration(None,
is_dict_config=True,
dict_config=config)
runner = DagRunner(configuration)
with pytest.raises(Exception) as e:
runner.filter_sequence(
["recipe_csv_writer", "read_data", "recipe_csv_writer2"])
assert (
"Traverser is mismatched with section writer_config. Expecting set ['recipe_csv_writer', 'recipe_csv_writer2']"
in str(e))
def test_filter_sequence2():
config = {
"implementation_config": {
"reader_config": {
"csv_reader": {
"class": "CsvReader",
"filename": "test/minimal.csv",
"destinations": [],
}
}
}
}
configuration = Configuration(None,
is_dict_config=True,
dict_config=config)
runner = DagRunner(configuration)
with pytest.raises(Exception) as e:
runner.filter_sequence(["csv_reader", "junk!"])
assert "Unknown key junk!" in str(e)
def test_filter_sequence1():
config = {
"implementation_config": {
"reader_config": {
"csv_reader": {
"class": "CsvReader",
"filename": "test/minimal.csv",
"destinations": [],
}
}
}
}
configuration = Configuration(None,
is_dict_config=True,
dict_config=config)
runner = DagRunner(configuration)
with pytest.raises(Exception) as e:
runner.filter_sequence(["csv_reader", "csv_reader"])
assert "You have duplicate nodes from traverser!" in str(e)
def test_filter_sequence3():
config = {
"implementation_config": {
"reader_config": {
"csv_reader": {
"class": "CsvReader",
"filename": "test/minimal.csv",
"destinations": [],
}
}
}
}
configuration = Configuration(None,
is_dict_config=True,
dict_config=config)
runner = DagRunner(configuration)
with pytest.raises(Exception) as e:
runner.filter_sequence([])
assert "Ran out of nodes for section reader_config. Only received []" in str(
e)