def test_filter_endpoint():
target1 = Target("Foo",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
target2 = Target("Bar",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
target3 = Target("FooBar",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
endpoint_filter = EndpointFilter(endpoints={target1})
assert set(endpoint_filter.apply([target1, target2, target3])) == {target1}
endpoint_filter = EndpointFilter(endpoints={target1, target3})
assert set(endpoint_filter.apply([target1, target2,
target3])) == {target1, target3}
endpoint_filter = EndpointFilter(endpoints={target1}, mode="exclude")
assert set(endpoint_filter.apply([target1, target2,
target3])) == {target2, target3}
endpoint_filter = EndpointFilter(endpoints={target1, target3},
mode="exclude")
assert set(endpoint_filter.apply([target1, target2, target3])) == {target2}
def test_filter_name():
target1 = Target("Foo",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
target2 = Target("Bar",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
target3 = Target("FooBar",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
name_filter = NameFilter(patterns=["Foo"])
assert set(name_filter.apply([target1, target2, target3])) == {target1}
name_filter = NameFilter(patterns=["Foo*"])
assert set(name_filter.apply([target1, target2,
target3])) == {target1, target3}
name_filter = NameFilter(patterns=["Foo", "Bar"])
assert set(name_filter.apply([target1, target2,
target3])) == {target1, target2}
def test_submit_1(popen):
t1 = Target.empty("Target1")
t2 = Target.empty("Target2")
t3 = Target.empty("Target3")
t4 = Target.empty("Target4")
popen.return_value.returncode = 0
popen.return_value.communicate.return_value = ("", "")
backend = SlurmBackend()
popen.return_value.returncode = 0
popen.return_value.communicate.side_effect = [
("1\n", ""),
("2\n", ""),
("3\n", ""),
("", "This is stderr\n"),
]
backend.submit(t1, [])
backend.submit(t2, [])
backend.submit(t3, [t1, t2])
assert (
call(
["/bin/sbatch", "--parsable"],
stderr=-1,
stdin=-1,
stdout=-1,
universal_newlines=True,
)
in popen.call_args_list
)
assert (
call(
["/bin/sbatch", "--parsable"],
stderr=-1,
stdin=-1,
stdout=-1,
universal_newlines=True,
)
in popen.call_args_list
)
assert (
call(
["/bin/sbatch", "--parsable", "--dependency=afterok:1:2"],
stderr=-1,
stdin=-1,
stdout=-1,
universal_newlines=True,
)
in popen.call_args_list
)
with pytest.raises(DependencyError):
backend.submit(t3, [t4])
popen.return_value.returncode = 1
with pytest.raises(BackendError):
backend.submit(t4, [])
def test_initialization(tmpdir, popen):
t1 = Target.empty("Target1")
t2 = Target.empty("Target2")
t3 = Target.empty("Target3")
t4 = Target.empty("Target4")
gwf_dir = tmpdir.ensure_dir(".gwf")
tracked_file = gwf_dir.join("slurm-backend-tracked.json")
tracked_file.write(
'{"Target1": "1", "Target2": "2", "Target3": "3", "Target4": "4"}'
)
popen.return_value.returncode = 0
popen.return_value.communicate.return_value = ("3;SE\n2;BF\n1;R\n", "")
with tmpdir.as_cwd():
backend = SlurmBackend()
assert backend.status(t1) == Status.RUNNING
assert backend.status(t2) == Status.UNKNOWN
assert backend.status(t3) == Status.SUBMITTED
assert backend.status(t4) == Status.UNKNOWN
popen.return_value.returncode = 1
popen.return_value.communicate.return_value = ("", "This is stderr")
with pytest.raises(BackendError):
with SlurmBackend():
pass
def test_submitting_target_correctly_sets_dependency_flag_for_sbatch(self):
target1 = Target(
'TestTarget1',
inputs=[],
outputs=['test_output1.txt'],
options={},
working_dir='/some/dir',
)
target2 = Target(
'TestTarget2',
inputs=[],
outputs=['test_output2.txt'],
options={},
working_dir='/some/dir',
)
target3 = Target(
'TestTarget3',
inputs=['test_output1.txt', 'test_output2.txt'],
outputs=['test_output3.txt'],
options={},
working_dir='/some/dir',
)
backend = SlurmBackend()
with self.force_job_id(1000):
backend.submit(target1, [])
with self.force_job_id(2000):
backend.submit(target2, [])
with self.force_job_id(3000):
backend.submit(target3, [target1, target2])
self.mock_call_sbatch.assert_any_call(ANY, ['1000', '2000'])
def test_scheduling_non_submitted_targets_that_should_not_run(backend, monkeypatch):
target1 = Target('TestTarget1', inputs=[], outputs=['test_output1.txt'], options={}, working_dir='/some/dir')
target2 = Target('TestTarget2', inputs=[], outputs=['test_output2.txt'], options={}, working_dir='/some/dir')
target3 = Target('TestTarget3', inputs=['test_output1.txt', 'test_output2.txt'], outputs=['test_output3.txt'], options={}, working_dir='/some/dir')
graph = Graph(targets={'TestTarget1': target1, 'TestTarget2': target2, 'TestTarget3': target3})
monkeypatch.setattr(graph, 'should_run', lambda t: False)
assert schedule(graph, backend, target3) == False
assert backend.submit.call_args_list == []
def test_scheduling_target_with_deps_that_are_not_submitted(backend, monkeypatch):
target1 = Target('TestTarget1', inputs=[], outputs=['test_output.txt'], options={}, working_dir='/some/dir')
target2 = Target('TestTarget2', inputs=['test_output.txt'], outputs=[], options={}, working_dir='/some/dir')
graph = Graph(targets={'TestTarget1': target1, 'TestTarget2': target2})
monkeypatch.setattr(graph, 'should_run', lambda t: True)
assert schedule(graph, backend, target2) == True
assert len(backend.submit.call_args_list) == 2
assert call(target1, dependencies=set()) in backend.submit.call_args_list
assert call(target2, dependencies=set([target1])) in backend.submit.call_args_list
def test_inherit_options(self):
target = Target(
"TestTarget",
inputs=[],
outputs=[],
options={"cores": 8},
working_dir="/some/dir",
)
target.inherit_options({"cores": 4, "memory": "4g"})
self.assertEqual(target.options, {"cores": 8, "memory": "4g"})
def setUp(self):
self.foobar = Target('foobar', inputs=[], outputs=[], options={}, working_dir='/some/dir')
self.foofoo = Target('foobar', inputs=[], outputs=[], options={}, working_dir='/some/dir')
self.barbar = Target('foobar', inputs=[], outputs=[], options={}, working_dir='/some/dir')
self.targets = {
'foobar': self.foobar,
'foofoo': self.foofoo,
'barbar': self.barbar,
}
def test_scheduling_many_targets_calls_schedule_for_each_target(backend, monkeypatch):
target1 = Target('TestTarget1', inputs=[], outputs=['test_output1.txt'], options={}, working_dir='/some/dir')
target2 = Target('TestTarget2', inputs=[], outputs=['test_output2.txt'], options={}, working_dir='/some/dir')
target3 = Target('TestTarget3', inputs=['test_output1.txt'], outputs=['test_output3.txt'], options={}, working_dir='/some/dir')
target4 = Target('TestTarget4', inputs=['test_output2.txt'], outputs=['test_output4.txt'], options={}, working_dir='/some/dir')
graph = Graph(targets={'TestTarget1': target1, 'TestTarget2': target2, 'TestTarget3': target3, 'TestTarget4': target4})
monkeypatch.setattr(graph, 'should_run', lambda t: True)
assert schedule_many(graph, backend, [target3, target4]) == [True, True]
assert call(target4, dependencies=set([target2])) in backend.submit.call_args_list
assert call(target3, dependencies=set([target1])) in backend.submit.call_args_list
assert call(target2, dependencies=set()) in backend.submit.call_args_list
assert call(target1, dependencies=set()) in backend.submit.call_args_list
def test_scheduling_branch_and_join_structure(backend, monkeypatch):
target1 = Target('TestTarget1', inputs=[], outputs=['output1.txt'], options={}, working_dir='/some/dir')
target2 = Target('TestTarget2', inputs=['output1.txt'], outputs=['output2.txt'], options={}, working_dir='/some/dir')
target3 = Target('TestTarget3', inputs=['output1.txt'], outputs=['output3.txt'], options={}, working_dir='/some/dir')
target4 = Target('TestTarget4', inputs=['output2.txt', 'output3.txt'], outputs=['final.txt'], options={}, working_dir='/some/dir')
graph = Graph(targets={'target1': target1, 'target2': target2, 'target3': target3, 'target4': target4})
monkeypatch.setattr(graph, 'should_run', lambda t: True)
assert schedule(graph, backend, target4) == True
assert len(backend.submit.call_args_list) == 4
assert call(target1, dependencies=set([])) in backend.submit.call_args_list
assert call(target2, dependencies=set([target1])) in backend.submit.call_args_list
assert call(target3, dependencies=set([target1])) in backend.submit.call_args_list
assert call(target4, dependencies=set([target3, target2])) in backend.submit.call_args_list
def test_scheduling_unsubmitted_target(backend, monkeypatch):
target = Target('TestTarget', inputs=[], outputs=[], options={}, working_dir='/some/dir')
graph = Graph(targets={'TestTarget': target})
monkeypatch.setattr(graph, 'should_run', lambda t: True)
assert schedule(graph, backend, target) == True
assert len(backend.submit.call_args_list) == 1
assert call(target, dependencies=set()) in backend.submit.call_args_list
def test_inject_target_defaults_into_target_options_on_submit(self):
target = Target('TestTarget',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
self.backend.submit(target, dependencies=[])
self.assertEqual(target.options, {'cores': 2, 'memory': '18g'})
target = Target('TestTarget',
inputs=[],
outputs=[],
options={'cores': 32},
working_dir='/some/dir')
self.backend.submit(target, dependencies=[])
self.assertEqual(target.options, {'cores': 32, 'memory': '18g'})
def test_submit_target_with_unknown_dependency_raises_unknown_dependency_error(
self):
target1 = Target('TestTarget1',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
target2 = Target('TestTarget2',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
backend = SlurmBackend()
with self.assertRaises(UnknownDependencyError):
backend.submit(target2, [target1])
def test_cancel(popen):
t = Target.empty("Target")
popen.return_value.returncode = 0
popen.return_value.communicate.return_value = ("", "")
backend = SlurmBackend()
with pytest.raises(TargetError):
backend.cancel(t)
popen.return_value.communicate.return_value = ("1\n", "")
backend.submit(t, [])
assert backend.status(t) == Status.SUBMITTED
backend.cancel(t)
assert backend.status(t) == Status.UNKNOWN
assert (
call(
["/bin/scancel", "--verbose", "1"],
stderr=-1,
stdin=-1,
stdout=-1,
universal_newlines=True,
)
in popen.call_args_list
)
popen.return_value.returncode = 1
popen.return_value.communicate.return_value = ("", "This is stderr")
with pytest.raises(BackendError):
backend.cancel(t)
def test_remove_options_with_none_value(self):
target = Target('TestTarget',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
self.backend.submit(target, dependencies=[])
assert target.options == {'cores': 2, 'memory': '18g'}
def test_should_stringify_output_paths(self):
target = Target(
name='TestTarget',
inputs=[],
outputs=[pathlib.PurePath('somefile.txt'), 'otherfile.txt'],
options={},
working_dir='/some/path')
self.assertEqual(target.outputs, ['/some/path/somefile.txt', '/some/path/otherfile.txt'])
def test_target_with_inputs_is_not_a_source(self):
target = Target(
name='TestTarget',
inputs=['test_input1.txt', 'test_input2.txt'],
outputs=[],
options={},
working_dir='/some/path'
)
self.assertFalse(target.is_source)
def test_submitted_should_only_return_true_if_target_is_in_job_db(self):
target1 = Target('TestTarget1',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
target2 = Target('TestTarget2',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
backend = SlurmBackend()
with self.force_job_id(1000):
backend.submit(target1, dependencies=[])
self.assertEqual(backend.status(target1), Status.SUBMITTED)
self.assertEqual(backend.status(target2), Status.UNKNOWN)
def test_target_with_invalid_name_raises_exception(self):
with self.assertRaises(InvalidNameError):
Target(
'123abc',
inputs=[],
outputs=[],
options={},
working_dir='/some/path'
)
def test_str_on_target(self):
target = Target(
'TestTarget',
inputs=[],
outputs=[],
options={},
working_dir='/some/path'
)
self.assertEqual(str(target), 'TestTarget')
def test_cancel_unknown_target_raises_exception(self):
target = Target('TestTarget',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
backend = SlurmBackend()
with self.assertRaises(UnknownTargetError):
backend.cancel(target)
def test_backend_submit_full_removes_options_with_none_value(backend):
target = Target(
"TestTarget",
inputs=[],
outputs=[],
options={"cores": None},
working_dir="/some/dir",
)
backend.submit_full(target, set())
assert target.options == {"memory": "1g"}
def test_backend_submit_full_injects_backend_defaults(backend):
target1 = Target("TestTarget1",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
target2 = Target(
"TestTarget2",
inputs=[],
outputs=[],
options={"cores": 32},
working_dir="/some/dir",
)
backend.submit_full(target1, set())
assert target1.options == {"cores": 1, "memory": "1g"}
backend.submit_full(target2, set())
assert target2.options == {"cores": 32, "memory": "1g"}
def test_target_should_run_if_it_is_a_sink(self):
target = Target('TestTarget', inputs=[], outputs=[], options={}, working_dir='/some/dir')
graph = Graph(targets={'TestTarget': target})
with self.assertLogs(level='DEBUG') as logs:
self.assertTrue(graph.should_run(target))
self.assertEqual(
logs.output,
[
'DEBUG:gwf.core:TestTarget should run because it is a sink.'
]
)
def test_warn_user_when_submitting_target_with_unsupported_option(self):
target = Target('TestTarget',
inputs=[],
outputs=[],
options={'foo': 'bar'},
working_dir='/some/dir')
with self.assertLogs(level='WARNING') as logs:
self.backend.submit(target, dependencies=[])
self.assertEqual(logs.output, [
'WARNING:gwf.backends.base:Option "foo" used in "TestTarget" is not supported by backend. Ignored.'
])
def test_absolute_output_paths_are_not_normalized(self):
target = Target(
name='TestTarget',
inputs=['test_output1.txt', '/other/path/test_output2.txt'],
outputs=[],
options={},
working_dir='/some/path'
)
self.assertTrue(target.inputs[0].startswith('/some/path'))
self.assertTrue(target.inputs[1].startswith('/other/path'))
def test_no_dependency_flag_is_set_if_target_has_no_dependencies(self):
target = Target('TestTarget',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
backend = SlurmBackend()
backend.submit(target, [])
self.mock_call_sbatch.assert_any_call(ANY, [])
def test_cancel_submitted_target(self):
target = Target('TestTarget',
inputs=[],
outputs=[],
options={},
working_dir='/some/dir')
backend = SlurmBackend()
with self.force_job_id(1000):
backend.submit(target, dependencies=[])
self.assertEqual(backend.status(target), Status.SUBMITTED)
backend.cancel(target)
self.assertEqual(backend.status(target), Status.UNKNOWN)
def test_relative_output_paths_are_normalized(self):
target = Target(
name='TestTarget',
inputs=[],
outputs=['test_output1.txt', 'test_output2.txt'],
options={},
working_dir='/some/path'
)
self.assertTrue(os.path.isabs(target.outputs[0]))
self.assertTrue(os.path.isabs(target.outputs[1]))
self.assertTrue(target.outputs[0].startswith('/some/path'))
self.assertTrue(target.outputs[1].startswith('/some/path'))
def test_executable_unavailable_after_initialization(popen, monkeypatch):
exes = {"squeue": "/bin/squeue"}
monkeypatch.setattr("gwf.backends.slurm.find_executable", lambda e: exes.get(e))
t = Target.empty("TestTarget")
popen.return_value.returncode = 0
popen.return_value.communicate.return_value = ("", "")
backend = SlurmBackend()
with pytest.raises(BackendError):
backend.cancel(t)
with pytest.raises(BackendError):
backend.submit(t, dependencies=[])
def test_backend_logs():
target = Target("TestTarget",
inputs=[],
outputs=[],
options={},
working_dir="/some/dir")
backend = TestingBackend()
with patch.object(backend.log_manager,
"open_stdout",
return_value=io.StringIO("foo")):
assert backend.logs(target).read() == "foo"
with patch.object(backend.log_manager,
"open_stderr",
return_value=io.StringIO("bar")):
assert backend.logs(target, stderr=True).read() == "bar"
def test_file_log_manager_remove(log_manager, tmpdir):
target = Target.empty("TestTarget")
stdout_file = tmpdir.join(".gwf", "logs", "TestTarget.stdout").ensure()
stderr_file = tmpdir.join(".gwf", "logs", "TestTarget.stderr").ensure()
log_manager.remove_stdout(target)
assert not stdout_file.exists()
log_manager.remove_stderr(target)
assert not stderr_file.exists()
with pytest.raises(LogError):
log_manager.remove_stdout(target)
with pytest.raises(LogError):
log_manager.remove_stderr(target)
def test_file_log_manager_open(log_manager, tmpdir):
target = Target.empty("TestTarget")
with pytest.raises(LogError):
log_manager.open_stdout(target)
with pytest.raises(LogError):
log_manager.open_stderr(target)
stdout_file = tmpdir.join(".gwf", "logs", "TestTarget.stdout")
stdout_file.write("This is standard output...")
stderr_file = tmpdir.join(".gwf", "logs", "TestTarget.stderr")
stderr_file.write("This is standard error...")
assert log_manager.open_stdout(target).read() == "This is standard output..."
assert log_manager.open_stderr(target).read() == "This is standard error..."
def test_file_log_manager_remove(log_manager, tmpdir):
target = Target.empty("TestTarget")
stdout_file = tmpdir.join(".gwf", "logs", "TestTarget.stdout").ensure()
stderr_file = tmpdir.join(".gwf", "logs", "TestTarget.stderr").ensure()
log_manager.remove_stdout(target)
assert not stdout_file.exists()
log_manager.remove_stderr(target)
assert not stderr_file.exists()
with pytest.raises(LogError):
log_manager.remove_stdout(target)
with pytest.raises(LogError):
log_manager.remove_stderr(target)
def test_backend_submit_full_warns_user_when_submitting_target_with_unsupported_option(
backend, caplog):
target = Target(
"TestTarget",
inputs=[],
outputs=[],
options={"foo": "bar"},
working_dir="/some/dir",
)
backend.submit_full(target, set())
assert target.options == {"cores": 1, "memory": "1g"}
assert caplog.record_tuples == [(
"gwf.backends.base",
logging.WARNING,
'Option "foo" used in "TestTarget" is not supported by backend. Ignored.',
)]
def test_file_log_manager_open(log_manager, tmpdir):
target = Target.empty("TestTarget")
with pytest.raises(LogError):
log_manager.open_stdout(target)
with pytest.raises(LogError):
log_manager.open_stderr(target)
stdout_file = tmpdir.join(".gwf", "logs", "TestTarget.stdout")
stdout_file.write("This is standard output...")
stderr_file = tmpdir.join(".gwf", "logs", "TestTarget.stderr")
stderr_file.write("This is standard error...")
assert log_manager.open_stdout(
target).read() == "This is standard output..."
assert log_manager.open_stderr(
target).read() == "This is standard error..."
def test_file_log_manager_path(log_manager):
target = Target.empty("TestTarget")
assert log_manager.stderr_path(target) == ".gwf/logs/TestTarget.stderr"
assert log_manager.stdout_path(target) == ".gwf/logs/TestTarget.stdout"