def test_including_workflow_instance_dispatches_to_include_workflow(self):
workflow = Workflow()
other_workflow = Workflow()
with patch.object(workflow, 'include_workflow', autospec=True) as mock_include_workflow:
workflow.include(other_workflow)
mock_include_workflow.assert_called_once_with(
other_workflow, namespace=None)
def test_raise_error_if_two_targets_in_different_namespaces_produce_the_same_file(self):
w1 = Workflow(name='foo')
w1.target('SayHello', inputs=[], outputs=['greeting.txt'])
w2 = Workflow(name='bar')
w2.target('SayHi', inputs=[], outputs=['greeting.txt'])
w2.include(w1)
with self.assertRaises(FileProvidedByMultipleTargetsError):
g = Graph(targets=w2.targets)
def test_including_workflow_from_path(self, mock_load_workflow):
workflow = Workflow()
target1 = workflow.target('TestTarget1', inputs=[], outputs=[])
other_workflow = Workflow()
target2 = other_workflow.target('TestTarget2', inputs=[], outputs=[])
target3 = other_workflow.target('TestTarget3', inputs=[], outputs=[])
mock_load_workflow.return_value = other_workflow
workflow.include_path('/path/to/other_workflow.py', namespace='other')
self.assertEqual(workflow.targets.keys(), {'TestTarget1', 'other.TestTarget2', 'other.TestTarget3'})
def test_including_workflow_module_gets_workflow_attribute_and_dispatches_to_include_workflow(self, mock_ismodule):
workflow = Workflow(working_dir='/some/dir')
other_workflow = Workflow(working_dir='/some/other/dir')
mock_module = Mock()
mock_module.gwf = other_workflow
with patch.object(workflow, 'include_workflow', autospec=True) as mock_include_workflow:
workflow.include(mock_module)
mock_ismodule.assert_called_once_with(mock_module)
mock_include_workflow.assert_called_once_with(
other_workflow, namespace=None)
def test_dependencies_correctly_resolved_for_named_workflow(self):
workflow = Workflow(name='foo')
target1 = workflow.target('TestTarget1', inputs=[], outputs=['test.txt'])
target2 = workflow.target('TestTarget2', inputs=['test.txt'], outputs=[])
other_workflow = Workflow(name='bar')
other_workflow.include(workflow)
other_target1 = other_workflow.target('TestTarget1', inputs=['test.txt'], outputs=[])
graph = Graph(targets=other_workflow.targets)
assert 'TestTarget1' in graph.targets
assert 'foo.TestTarget2' in graph.targets
assert 'foo.TestTarget2' in graph.targets
def test_including_workflow_object_should_extend_including_workflow(self):
workflow = Workflow()
workflow.target('TestTarget1', inputs=[], outputs=[])
other_workflow = Workflow(name='foo')
other_workflow.target('TestTarget2', inputs=[], outputs=[])
other_workflow.target('TestTarget3', inputs=[], outputs=[])
workflow.include_workflow(other_workflow)
self.assertIn('TestTarget1', workflow.targets)
self.assertIn('foo.TestTarget2', workflow.targets)
self.assertIn('foo.TestTarget3', workflow.targets)
def test_including_workflow_object_should_extend_including_workflow():
workflow = Workflow()
workflow.target("TestTarget1", inputs=[], outputs=[])
other_workflow = Workflow(name="foo")
other_workflow.target("TestTarget2", inputs=[], outputs=[])
other_workflow.target("TestTarget3", inputs=[], outputs=[])
workflow.include_workflow(other_workflow)
assert "TestTarget1" in workflow.targets
assert "foo.TestTarget2" in workflow.targets
assert "foo.TestTarget3" in workflow.targets
def test_include_target_from_workflow_in_two_different_workflows_(self):
w1 = Workflow()
target = w1.target('MyTarget', inputs=[], outputs=[])
w3 = Workflow()
w3.include(w1, namespace='bar')
w2 = Workflow()
w2.include(w1, namespace='foo')
self.assertEqual(target.name, 'MyTarget')
self.assertIn('bar.MyTarget', w3.targets)
self.assertEqual(w3.targets['bar.MyTarget'].name, 'bar.MyTarget')
self.assertIn('foo.MyTarget', w2.targets)
self.assertEqual(w2.targets['foo.MyTarget'].name, 'foo.MyTarget')
def test_include_target_from_workflow_in_two_different_workflows_():
w1 = Workflow()
target = w1.target("MyTarget", inputs=[], outputs=[])
w3 = Workflow()
w3.include(w1, namespace="bar")
w2 = Workflow()
w2.include(w1, namespace="foo")
assert target.name == "MyTarget"
assert "bar.MyTarget" in w3.targets
assert w3.targets["bar.MyTarget"].name == "bar.MyTarget"
assert "foo.MyTarget" in w2.targets
assert w2.targets["foo.MyTarget"].name == "foo.MyTarget"
def test_workflow_computes_working_dir_when_not_initialized_with_working_dir(
self, inspect_getfile_mock, sys_getframe_mock):
workflow = Workflow()
self.assertEqual(sys_getframe_mock.call_count, 1)
self.assertEqual(inspect_getfile_mock.call_count, 1)
self.assertEqual(workflow.working_dir, '/some/path')
def test_including_workflow_path_dispatches_to_include_path(self):
workflow = Workflow()
with patch.object(workflow, 'include_path', autospec=True) as mock_include_path:
workflow.include('/path/to/other_workflow.py')
mock_include_path.assert_called_once_with(
'/path/to/other_workflow.py', namespace=None)
def test_workflow_computes_working_dir_when_not_initialized_with_working_dir(
inspect_getfile_mock, sys_getframe_mock):
workflow = Workflow()
assert sys_getframe_mock.call_count == 1
assert inspect_getfile_mock.call_count == 1
assert workflow.working_dir == "/some/path"
def test_map_naming_with_invalid_template_arg():
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
with pytest.raises(ValueError):
workflow.map(42, files)
def test_two_targets_producing_the_same_file_but_declared_with_rel_and_abs_path(self, mock_os_path_exists):
workflow = Workflow(working_dir='/some/dir')
workflow.target('TestTarget1', inputs=[], outputs=['/some/dir/test_output.txt'])
workflow.target('TestTarget2', inputs=[], outputs=['test_output.txt'])
with self.assertRaises(FileProvidedByMultipleTargetsError):
Graph(targets=workflow.targets)
def test_target_list():
def my_template(path):
return AnonymousTarget(inputs={"path": path},
outputs={"path": path + ".new"},
options={})
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
target_list = workflow.map(my_template, files)
assert len(target_list) == 3
assert len(target_list.outputs) == 3
assert target_list.outputs == [
{
"path": "a.new"
},
{
"path": "b.new"
},
{
"path": "c.new"
},
]
assert len(target_list.inputs) == 3
assert target_list.inputs == [{"path": "a"}, {"path": "b"}, {"path": "c"}]
def test_including_workflow_from_path(mock_load_workflow):
workflow = Workflow()
workflow.target("TestTarget1", inputs=[], outputs=[])
other_workflow = Workflow()
other_workflow.target("TestTarget2", inputs=[], outputs=[])
other_workflow.target("TestTarget3", inputs=[], outputs=[])
mock_load_workflow.return_value = other_workflow
workflow.include_path("/path/to/other_workflow.py", namespace="other")
assert workflow.targets.keys() == {
"TestTarget1",
"other.TestTarget2",
"other.TestTarget3",
}
def test_map_arg_passing_list_of_dicts(mocker, mock_template):
files = [
{
"path": "a",
"output_dir": "foo/"
},
{
"path": "b",
"output_dir": "foo/"
},
{
"path": "c",
"output_dir": "foo/"
},
]
workflow = Workflow(working_dir="/some/dir")
workflow.map(mock_template, files)
mock_template.assert_has_calls(
[
mocker.call(path="a", output_dir="foo/"),
mocker.call(path="b", output_dir="foo/"),
mocker.call(path="c", output_dir="foo/"),
],
any_order=True,
)
def test_shell_calls_subprocess_with_same_working_dir_as_workflow_in_a_shell(
mock_check_output, ):
workflow = Workflow(working_dir="/some/path")
workflow.shell("echo hello")
mock_check_output.assert_called_once_with("echo hello",
cwd="/some/path",
shell=True)
def test_target_from_template_returning_tuple():
def template_returning_tuple():
return [], [], {}, "this is the spec"
workflow = Workflow(working_dir="/some/dir")
workflow.target_from_template("TestTarget", template_returning_tuple())
assert "TestTarget" in workflow.targets
def setUp(self):
workflow = Workflow(working_dir='/some/dir')
self.target1 = workflow.target('TestTarget1', inputs=[], outputs=['test_output1.txt'])
self.target2 = workflow.target('TestTarget2', inputs=['test_output1.txt'], outputs=['test_output2.txt'])
self.target3 = workflow.target('TestTarget3', inputs=['test_output1.txt'], outputs=['test_output3.txt'])
self.target4 = workflow.target('TestTarget4', inputs=['test_output2.txt', 'test_output3.txt'], outputs=['final_output.txt'])
self.graph = Graph(targets=workflow.targets)
def test_including_workflow_path_dispatches_to_include_path():
workflow = Workflow()
with patch.object(workflow, "include_path",
autospec=True) as mock_include_path:
workflow.include("/path/to/other_workflow.py")
mock_include_path.assert_called_once_with("/path/to/other_workflow.py",
namespace=None)
def main():
gwf = Workflow()
working_directory = '/home/andyb/CUP_classification/faststorage/Andrej'
reference_genome = f'{working_directory}/inputs/hg38.fa'
gwf.target_from_template(
'create_index',
create_bowtie2_index(reference_genome, cores=8, memory='16g'))
def test_map_arg_passing_list_of_strings(mocker, mock_template):
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
workflow.map(mock_template, files)
mock_template.assert_has_calls(
[mocker.call("a"),
mocker.call("b"),
mocker.call("c")], any_order=True)
def test_target_from_invalid_template():
def invalid_template():
return [], []
workflow = Workflow()
with pytest.raises(TypeError):
workflow.target_from_template("TestTarget", 50)
with pytest.raises(TypeError):
workflow.target_from_template("TestTarget", invalid_template())
def test_target_from_template_returning_anonymous_target_without_working_dir():
def template_returning_anonymous_target_without_working_dir():
return AnonymousTarget(inputs=["hello.txt"],
outputs=[],
options={},
spec="this is the spec")
workflow = Workflow(working_dir="/some/dir")
workflow.target_from_template(
"TestTarget",
template_returning_anonymous_target_without_working_dir())
assert "TestTarget" in workflow.targets
def test_map_with_custom_naming_string():
def my_template(path):
return AnonymousTarget(inputs={"path": path},
outputs={"path": path + ".new"},
options={})
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
workflow.map(my_template, files, name="bar")
assert len(workflow.targets) == 3
assert "bar_0" in workflow.targets
assert "bar_1" in workflow.targets
assert "bar_2" in workflow.targets
def test_map_naming_with_template_function():
def my_template(path):
return AnonymousTarget(inputs=[path],
outputs=[path + ".new"],
options={})
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
workflow.map(my_template, files)
assert len(workflow.targets) == 3
assert "my_template_0" in workflow.targets
assert "my_template_1" in workflow.targets
assert "my_template_2" in workflow.targets
def test_map_naming_with_template_class_instance():
class MyTemplate:
def __call__(self, path):
return AnonymousTarget(inputs=[path],
outputs=[path + ".new"],
options={})
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
workflow.map(MyTemplate(), files)
assert len(workflow.targets) == 3
assert "MyTemplate_0" in workflow.targets
assert "MyTemplate_1" in workflow.targets
assert "MyTemplate_2" in workflow.targets
def test_map_with_custom_naming_function():
def my_template(path):
return AnonymousTarget(inputs={"path": path},
outputs={"path": path + ".new"},
options={})
files = ["a", "b", "c"]
workflow = Workflow(working_dir="/some/dir")
workflow.map(my_template,
files,
name=lambda i, t: "foo_{}".format(t.inputs["path"]))
assert len(workflow.targets) == 3
assert "foo_a" in workflow.targets
assert "foo_b" in workflow.targets
assert "foo_c" in workflow.targets
def test_target_should_not_run_if_it_is_a_source_and_all_outputs_exist(self):
workflow = Workflow(working_dir='/some/dir')
target = workflow.target(
'TestTarget1',
inputs=[],
outputs=['test_output1.txt', 'test_output2.txt']
)
graph = Graph(targets=workflow.targets)
mock_file_cache = {
'/some/dir/test_output1.txt': 1,
'/some/dir/test_output2.txt': 2
}
with patch.dict(graph.file_cache, mock_file_cache):
self.assertFalse(
graph.should_run(target)
)