def test_write_to_stdin(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
process = Popen('a command', stdin=PIPE, shell=True)
process.stdin.write('some text')
# test call list
compare(Popen.mock.method_calls,
expected=[
call.Popen('a command', shell=True, stdin=PIPE),
call.Popen_instance.stdin.write('some text'),
])
compare(Popen.all_calls,
expected=[
call.Popen('a command', shell=True, stdin=PIPE),
call.Popen('a command', shell=True,
stdin=PIPE).stdin.write('some text'),
])
compare(process.mock.method_calls,
expected=[
call.stdin.write('some text'),
])
compare(process.calls, expected=[
call.stdin.write('some text'),
])
repr(call.stdin.write('some text'))
def test_simultaneous_processes(self):
Popen = MockPopen()
Popen.set_command('a command', b'a', returncode=1)
Popen.set_command('b command', b'b', returncode=2)
process_a = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
process_b = Popen(['b', 'command'],
stdout=PIPE,
stderr=PIPE,
shell=True)
compare(process_a.wait(), expected=1)
compare(process_b.wait(), expected=2)
a_call = call.Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
b_call = call.Popen(['b', 'command'],
stdout=PIPE,
stderr=PIPE,
shell=True)
compare(Popen.all_calls,
expected=[
a_call,
b_call,
a_call.wait(),
b_call.wait(),
])
compare(process_a.mock.method_calls, expected=[call.wait()])
compare(process_b.mock.method_calls, expected=[call.wait()])
def test_submit_job_failure(self, mock_kill):
self.Popen.set_command(
'sbatch --wait --job-name popper_1_123abc '
'--output /tmp/popper/slurm/popper_1_123abc.out '
'/tmp/popper/slurm/popper_1_123abc.sh',
returncode=12)
self.Popen.set_command('tail -f /tmp/popper/slurm/popper_1_123abc.out',
returncode=0)
config_dict = {
'engine': {
'name': 'docker',
'options': {}
},
'resource_manager': {
'name': 'slurm',
'options': {}
}
}
config = PopperConfig(workspace_dir='/w', config_file=config_dict)
config.wid = "123abc"
with WorkflowRunner(config) as r:
wf = YMLWorkflow("""
version: '1'
steps:
- uses: 'popperized/bin/sh@master'
runs: [cat]
args: README.md
""")
wf.parse()
self.assertRaises(SystemExit, r.run, wf)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_1_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_1_123abc',
'--output', '/tmp/popper/slurm/popper_1_123abc.out',
'/tmp/popper/slurm/popper_1_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_submit_batch_job(self, mock_kill):
config = ConfigLoader.load(workspace_dir="/w")
self.Popen.set_command(
"sbatch --wait "
f"--job-name popper_sample_{config.wid} "
f"--output {slurm_cache_dir}/popper_sample_{config.wid}.out "
f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
returncode=0,
)
self.Popen.set_command(
f"tail -f {slurm_cache_dir}/popper_sample_{config.wid}.out",
returncode=0)
step = Box({"id": "sample"}, default_box=True)
with SlurmRunner(config=config) as sr:
sr._submit_batch_job(["ls -la"], step)
with open(f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
"r") as f:
content = f.read()
self.assertEqual(content, "#!/bin/bash\nls -la")
self.assertEqual(len(sr._spawned_jobs), 0)
self.assertEqual(sr._out_stream_thread.is_alive(), False)
call_tail = call.Popen(
[
"tail", "-f",
f"{slurm_cache_dir}/popper_sample_{config.wid}.out"
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
call_sbatch = call.Popen(
[
"sbatch",
"--wait",
"--job-name",
f"popper_sample_{config.wid}",
"--output",
f"{slurm_cache_dir}/popper_sample_{config.wid}.out",
f"{slurm_cache_dir}/popper_sample_{config.wid}.sh",
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_wait_and_return_code(self):
# setup
Popen = MockPopen()
Popen.set_command('a command', returncode=3)
# usage
process = Popen('a command')
compare(process.returncode, None)
# result checking
compare(process.wait(), 3)
compare(process.returncode, 3)
# test call list
compare(Popen.all_calls, expected=[
call.Popen('a command'),
call.Popen('a command').wait(),
])
def test_multiple_uses(self):
Popen = MockPopen()
Popen.set_command('a command', b'a')
Popen.set_command('b command', b'b')
process = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
out, err = process.communicate('foo')
compare(out, b'a')
process = Popen(['b', 'command'], stdout=PIPE, stderr=PIPE, shell=True)
out, err = process.communicate('foo')
compare(out, b'b')
compare([
call.Popen('a command', shell=True, stderr=-1, stdout=-1),
call.Popen_instance.communicate('foo'),
call.Popen(['b', 'command'], shell=True, stderr=-1, stdout=-1),
call.Popen_instance.communicate('foo'),
], Popen.mock.method_calls)
def test_multiple_processes(self):
# set up
self.Popen.set_command('process --batch=0', stdout=b'42')
self.Popen.set_command('process --batch=1', stdout=b'13')
# testing of results
compare(process_in_batches(2), expected=55)
# testing of process management:
p1 = call.Popen('process --batch=0', shell=True, stderr=PIPE, stdout=PIPE)
p2 = call.Popen('process --batch=1', shell=True, stderr=PIPE, stdout=PIPE)
compare(Popen.all_calls, expected=[
p1,
p2,
p1.communicate(),
p2.communicate(),
])
def test_submit_batch_job(self, mock_kill):
self.Popen.set_command(
'sbatch --wait '
'--job-name popper_sample_123abc '
'--output /tmp/popper/slurm/popper_sample_123abc.out '
'/tmp/popper/slurm/popper_sample_123abc.sh',
returncode=0)
self.Popen.set_command(
'tail -f /tmp/popper/slurm/popper_sample_123abc.out', returncode=0)
config = PopperConfig(workspace_dir='/w')
config.wid = "123abc"
step = {"name": "sample"}
with SlurmRunner(config=config) as sr:
sr._submit_batch_job(["ls -la"], step)
with open("/tmp/popper/slurm/popper_sample_123abc.sh", 'r') as f:
content = f.read()
self.assertEqual(content, "#!/bin/bash\nls -la")
self.assertEqual(len(sr._spawned_jobs), 0)
self.assertEqual(sr._out_stream_thread.is_alive(), False)
call_tail = call.Popen(
['tail', '-f', '/tmp/popper/slurm/popper_sample_123abc.out'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
call_sbatch = call.Popen([
'sbatch', '--wait', '--job-name', 'popper_sample_123abc',
'--output', '/tmp/popper/slurm/popper_sample_123abc.out',
'/tmp/popper/slurm/popper_sample_123abc.sh'
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True)
self.assertEqual(call_tail in self.Popen.all_calls, True)
self.assertEqual(call_sbatch in self.Popen.all_calls, True)
def test_start_new_session(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
Popen('a command', start_new_session=True)
# test call list
compare([
call.Popen('a command', start_new_session=True),
], Popen.mock.method_calls)
def test_command_is_sequence(self):
Popen = MockPopen()
Popen.set_command('a command')
process = Popen(['a', 'command'], stdout=PIPE, stderr=PIPE)
compare(process.wait(), 0)
compare([
call.Popen(['a', 'command'], stderr=-1, stdout=-1),
call.Popen_instance.wait(),
], Popen.mock.method_calls)
def test_example(self):
# set up
self.Popen.set_command('svn ls -R foo', stdout=b'o', stderr=b'e')
# testing of results
compare(my_func(), b'o')
# testing calls were in the right order and with the correct parameters:
compare([
call.Popen('svn ls -R foo', shell=True, stderr=PIPE, stdout=PIPE),
call.Popen_instance.communicate()
], Popen.mock.method_calls)
def test_communicate_with_input(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
process = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
out, err = process.communicate('foo')
# test call list
compare([
call.Popen('a command', shell=True, stderr=-1, stdout=-1),
call.Popen_instance.communicate('foo'),
], Popen.mock.method_calls)
def test_read_from_stdout_and_stderr(self):
# setup
Popen = MockPopen()
Popen.set_command('a command', stdout=b'foo', stderr=b'bar')
# usage
process = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
compare(process.stdout.read(), b'foo')
compare(process.stderr.read(), b'bar')
# test call list
compare([
call.Popen('a command', shell=True, stderr=PIPE, stdout=PIPE),
], Popen.mock.method_calls)
def test_kill(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
process = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
process.kill()
# result checking
compare([
call.Popen('a command', shell=True, stderr=-1, stdout=-1),
call.Popen_instance.kill(),
], Popen.mock.method_calls)
def test_read_from_stderr(self):
# setup
Popen = MockPopen()
Popen.set_command('a command', stderr=b'foo')
# usage
process = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
self.assertTrue(isinstance(process.stdout.fileno(), int))
compare(process.stderr.read(), b'foo')
# test call list
compare([
call.Popen('a command', shell=True, stderr=-1, stdout=-1),
], Popen.mock.method_calls)
def test_default_behaviour(self):
# set up
self.Popen.set_default(stdout=b'o', stderr=b'e')
# testing of results
compare(my_func(), b'o')
# testing calls were in the right order and with the correct parameters:
root_call = call.Popen('svn ls -R foo',
shell=True,
stderr=PIPE,
stdout=PIPE)
compare(Popen.all_calls, expected=[root_call, root_call.communicate()])
def test_example(self):
# set up
self.Popen.set_command('svn ls -R foo', stdout=b'o', stderr=b'e')
# testing of results
compare(my_func(), b'o')
# testing calls were in the right order and with the correct parameters:
process = call.Popen('svn ls -R foo',
shell=True,
stderr=PIPE,
stdout=PIPE)
compare(Popen.all_calls, expected=[process, process.communicate()])
def test_exec_srun(self, mock_kill):
config_dict = {
"engine": {
"name": "singularity",
"options": {},
},
"resource_manager": {
"name": "slurm",
"options": {
"sample": {
"gpus-per-task": 2,
"overcommit": True
}
},
},
}
config = ConfigLoader.load(workspace_dir="/w", config_file=config_dict)
self.Popen.set_command(
"srun --nodes 1 --ntasks 1 --ntasks-per-node 1 --gpus-per-task 2 --overcommit ls -la",
returncode=0,
)
step = Box({"id": "sample"}, default_box=True)
with SlurmRunner(config=config) as sr:
e = sr._exec_srun(["ls", "-la"], step, logging=True)
self.assertEqual(e, 0)
call_srun = call.Popen(
[
"srun",
"--nodes",
"1",
"--ntasks",
"1",
"--ntasks-per-node",
"1",
"--gpus-per-task",
"2",
"--overcommit",
"ls",
"-la",
],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
)
self.assertEqual(call_srun in self.Popen.all_calls, True)
def test_write_to_stdin(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
process = Popen('a command', stdin=PIPE, shell=True)
process.stdin.write('some text')
process.stdin.close()
# test call list
compare([
call.Popen('a command', shell=True, stdin=PIPE),
call.Popen_instance.stdin.write('some text'),
call.Popen_instance.stdin.close(),
], Popen.mock.method_calls)
def test_stop_running_tasks(self):
self.Popen.set_command('scancel --name job_a', returncode=0)
with SlurmRunner(config=PopperConfig()) as sr:
sr._spawned_jobs.add('job_a')
sr.stop_running_tasks()
self.assertEqual(
call.Popen(['scancel', '--name', 'job_a'],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True) in self.Popen.all_calls,
True)
def test_callable(self):
# set up
def command_callable(command, stdin):
return PopenBehaviour(stdout=b'stdout')
self.Popen.set_default(behaviour=command_callable)
# testing of results
compare(my_func(), b'stdout')
# testing calls were in the right order and with the correct parameters:
compare([
call.Popen('svn ls -R foo', shell=True, stderr=PIPE, stdout=PIPE),
call.Popen_instance.communicate()
], Popen.mock.method_calls)
def test_wait_and_return_code(self):
# setup
Popen = MockPopen()
Popen.set_command('a command', returncode=3)
# usage
process = Popen('a command')
compare(process.returncode, None)
# result checking
compare(process.wait(), 3)
compare(process.returncode, 3)
# test call list
compare([
call.Popen('a command'),
call.Popen_instance.wait(),
], Popen.mock.method_calls)
def test_all_signals(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
process = Popen('a command')
process.send_signal(signal.SIGINT)
process.terminate()
process.kill()
# test call list
compare([
call.Popen('a command'),
call.Popen_instance.send_signal(signal.SIGINT),
call.Popen_instance.terminate(),
call.Popen_instance.kill(),
], Popen.mock.method_calls)
def test_stop_running_tasks(self):
self.Popen.set_command("scancel --name job_a", returncode=0)
with SlurmRunner(config=ConfigLoader.load()) as sr:
sr._spawned_jobs.add("job_a")
sr.stop_running_tasks()
compare(
call.Popen(
["scancel", "--name", "job_a"],
cwd=os.getcwd(),
env=None,
preexec_fn=os.setsid,
stderr=-2,
stdout=-1,
universal_newlines=True,
),
self.Popen.all_calls[0],
)
def test_wait_timeout(self):
Popen = MockPopen()
Popen.set_command('a command', returncode=3)
process = Popen('a command')
if PY2:
with ShouldRaise(TypeError):
process.wait(timeout=1)
with ShouldRaise(TypeError):
process.wait(1)
else:
process.wait(timeout=1)
process.wait(1)
compare([
call.Popen('a command'),
call.Popen_instance.wait(timeout=1),
call.Popen_instance.wait(1)
],
expected=Popen.mock.method_calls)
def test_poll_setup(self):
# setup
Popen = MockPopen()
Popen.set_command('a command', poll_count=1)
# usage
process = Popen('a command', stdout=PIPE, stderr=PIPE, shell=True)
compare(process.poll(), None)
compare(process.poll(), 0)
compare(process.wait(), 0)
compare(process.poll(), 0)
# result checking
compare([
call.Popen('a command', shell=True, stderr=-1, stdout=-1),
call.Popen_instance.poll(),
call.Popen_instance.poll(),
call.Popen_instance.wait(),
call.Popen_instance.poll(),
], Popen.mock.method_calls)
def test_non_pipe(self):
# setup
Popen = MockPopen()
Popen.set_command('a command')
# usage
process = Popen('a command')
# checks
compare(process.stdout, expected=None)
compare(process.stderr, expected=None)
out, err = process.communicate()
# test the rest
compare(out, expected=None)
compare(err, expected=None)
# test call list
compare([
call.Popen('a command'),
call.Popen_instance.communicate(),
], Popen.mock.method_calls)
def test_poll_until_result(self):
# setup
Popen = MockPopen()
Popen.set_command('a command', returncode=3, poll_count=2)
# example usage
process = Popen('a command')
while process.poll() is None:
# you'd probably have a sleep here, or go off and
# do some other work.
pass
# result checking
compare(process.returncode, 3)
compare([
call.Popen('a command'),
call.Popen_instance.poll(),
call.Popen_instance.poll(),
call.Popen_instance.poll(),
], Popen.mock.method_calls)
def test_default_command_max_args(self):
Popen = MockPopen()
Popen.set_default(b'out', b'err', 1, 345)
process = Popen('a command', stdout=PIPE, stderr=PIPE)
compare(process.pid, 345)
compare(None, process.returncode)
out, err = process.communicate()
# test the rest
compare(out, b'out')
compare(err, b'err')
compare(process.returncode, 1)
# test call list
compare([
call.Popen('a command', stderr=-1, stdout=-1),
call.Popen_instance.communicate(),
], Popen.mock.method_calls)
def test_default_command_min_args(self):
# setup
Popen = MockPopen()
Popen.set_default()
# usage
process = Popen('a command', stdout=PIPE, stderr=PIPE)
# process started, no return code
compare(process.pid, 1234)
compare(None, process.returncode)
out, err = process.communicate()
# test the rest
compare(out, b'')
compare(err, b'')
compare(process.returncode, 0)
# test call list
compare([
call.Popen('a command', stderr=-1, stdout=-1),
call.Popen_instance.communicate(),
], Popen.mock.method_calls)
