def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def test_sequential_commands__atomiccmds():
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.run = mock.run_1
cmd_1.join = mock.join_1
cmd_1.join.return_value = [0]
cmd_2 = AtomicCmd(["ls"])
cmd_2.run = mock.run_2
cmd_2.join = mock.join_2
cmd_2.join.return_value = [0]
cmd_3 = AtomicCmd(["ls"])
cmd_3.run = mock.run_3
cmd_3.join = mock.join_3
cmd_3.join.return_value = [0]
cmds = SequentialCmds((cmd_1, cmd_2, cmd_3))
assert not cmds.ready()
cmds.run("xTMPx")
assert cmds.ready()
assert cmds.join() == [0, 0, 0]
assert mock.mock_calls == [
call.run_1("xTMPx"),
call.join_1(),
call.run_2("xTMPx"),
call.join_2(),
call.run_3("xTMPx"),
call.join_3(),
call.join_1(),
call.join_2(),
call.join_3(),
]
def test_atomiccmd__commit_before_join(temp_folder):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.join()
def test_atomiccmd__commit_before_join(temp_folder):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.join()
def test_atomiccmd__ready(tmp_path):
cmd = AtomicCmd("ls")
assert cmd.join() == [None]
assert not cmd.ready()
cmd.run(tmp_path)
assert cmd.join() == [0]
assert cmd.ready()
def test_atomiccmd__commit_while_running(tmp_path):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(tmp_path)
with pytest.raises(CmdError):
cmd.commit(tmp_path)
cmd.terminate()
cmd.join()
def test_atomiccmd__run__already_running(tmp_path):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(tmp_path)
with pytest.raises(CmdError):
cmd.run(tmp_path)
cmd.terminate()
cmd.join()
def test_atomiccmd__commit_before_join(tmp_path):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(tmp_path)
while cmd._proc.poll() is None:
pass
with pytest.raises(CmdError):
cmd.commit(tmp_path)
cmd.join()
def test_atomiccmd__commit_missing_files(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"),
OUT_BAR=os.path.join(destination, "4567"))
cmd.run(temp_folder)
cmd.join()
before = set(os.listdir(temp_folder))
assert_raises(CmdError, cmd.commit, temp_folder)
assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__piping_is_only_allowed_once(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"], OUT_STDOUT=AtomicCmd.PIPE)
cmd_2a = AtomicCmd(["grep", "foo"], IN_STDIN=cmd_1)
cmd_2b = AtomicCmd(["grep", "bar"], IN_STDIN=cmd_1)
cmd_1.run(temp_folder)
cmd_2a.run(temp_folder)
assert_raises(CmdError, cmd_2b.run, temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2a.join(), [0])
assert_equal(cmd_2b.join(), [None])
def test_atomiccmd__commit_missing_files(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"),
OUT_BAR=os.path.join(destination, "4567"))
cmd.run(temp_folder)
cmd.join()
before = set(os.listdir(temp_folder))
assert_raises(CmdError, cmd.commit, temp_folder)
assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def test_pformat__simple__running__set_cwd(tmp_path):
cmd = AtomicCmd(("sleep", "10"), set_cwd=True)
cmd.run(tmp_path)
assert pformat(cmd) == ("Command = sleep 10\n"
"Status = Running\n"
"STDOUT* = 'pipe_sleep_{id}.stdout'\n"
"STDERR* = 'pipe_sleep_{id}.stderr'\n"
"CWD = '{temp_dir}'").format(id=id(cmd),
temp_dir=tmp_path)
cmd.terminate()
cmd.join()
def test_pformat__simple__running__set_cwd(temp_folder):
cmd = AtomicCmd(("sleep", "10"), set_cwd=True)
cmd.run(temp_folder)
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Running ...\n"
" STDOUT* = 'pipe_sleep_{id}.stdout'\n"
" STDERR* = 'pipe_sleep_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
cmd.terminate()
cmd.join()
def test_atomiccmd__piping_temp(tmp_path):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE)
assert cmd_1.output_files == frozenset()
cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt")
assert cmd_2.input_files == frozenset()
cmd_1.run(tmp_path)
cmd_2.run(tmp_path)
assert cmd_1.join() == [0]
assert cmd_2.join() == [0]
result = (tmp_path / "piped.txt").read_text()
assert result == "#@!$^"
def test_atomiccmd__piping_is_only_allowed_once(tmp_path):
cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"], OUT_STDOUT=AtomicCmd.PIPE)
cmd_2a = AtomicCmd(["grep", "foo"], IN_STDIN=cmd_1)
cmd_2b = AtomicCmd(["grep", "bar"], IN_STDIN=cmd_1)
cmd_1.run(tmp_path)
cmd_2a.run(tmp_path)
with pytest.raises(CmdError):
cmd_2b.run(tmp_path)
assert cmd_1.join() == [0]
assert cmd_2a.join() == [0]
assert cmd_2b.join() == [None]
def test_pformat__simple__running(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Running ...\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
cmd.terminate()
cmd.join()
def test_parallel_commands__join_before_run():
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.join = mock.join_1
cmd_2 = AtomicCmd(["ls"])
cmd_2.join = mock.join_2
cmd_3 = AtomicCmd(["ls"])
cmd_3.join = mock.join_3
cmds = ParallelCmds((cmd_3, cmd_2, cmd_1))
assert cmds.join() == [None, None, None]
assert mock.mock_calls == []
def test_atomiccmd__commit_missing_files(tmp_path):
destination, tmp_path = _setup_for_commit(tmp_path, False)
cmd = AtomicCmd(
("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"),
OUT_BAR=os.path.join(destination, "4567"),
)
cmd.run(tmp_path)
cmd.join()
before = set(os.listdir(tmp_path))
with pytest.raises(CmdError):
cmd.commit(tmp_path)
assert before == set(os.listdir(tmp_path))
def test_atomiccmd__piping_is_only_allowed_once(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"],
OUT_STDOUT=AtomicCmd.PIPE)
cmd_2a = AtomicCmd(["grep", "foo"],
IN_STDIN=cmd_1)
cmd_2b = AtomicCmd(["grep", "bar"],
IN_STDIN=cmd_1)
cmd_1.run(temp_folder)
cmd_2a.run(temp_folder)
assert_raises(CmdError, cmd_2b.run, temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2a.join(), [0])
assert_equal(cmd_2b.join(), [None])
def test_pformat__simple__running(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Running ...\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
cmd.terminate()
cmd.join()
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"],
TEMP_OUT_STDOUT=AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"],
TEMP_IN_STDIN=cmd_1,
OUT_STDOUT="piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def test_atomiccmd__commit_temp_only(tmp_path):
cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt")
cmd.run(tmp_path)
assert cmd.join() == [0]
assert os.path.exists(os.path.join(tmp_path, "bar.txt"))
cmd.commit(tmp_path)
assert os.listdir(tmp_path) == []
def test_atomiccmd__terminate_race_condition(tmp_path):
cmd = AtomicCmd("true")
cmd.run(tmp_path)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert cmd.join() == [0]
def test_atomiccmd__terminate_race_condition(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__paths__key(temp_folder):
cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"),
OUT_STDOUT=AtomicCmd.PIPE)
cmd.run(temp_folder)
path = cmd._proc.stdout.read()
assert os.path.samefile(temp_folder, path), (temp_folder, path)
assert_equal(cmd.join(), [0])
def test_atomiccmd__paths_non_str(temp_folder):
cmd = AtomicCmd(("touch", 1234),
OUT_FOO="1234",
set_cwd=True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "1234"))
def test_atomiccmd__terminate_race_condition(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"),
TEMP_OUT_STDOUT="bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd))))
result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd))))
assert_equal(result_out, "STDOUT!")
assert_equal(result_err, "STDERR!")
def test_atomiccmd__pipes_stdin(temp_folder):
fname = test_file("fasta_file.fasta")
cmd = AtomicCmd("cat", IN_STDIN=fname, OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result,
">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__commit_with_pipes(tmp_path):
destination, tmp_path = _setup_for_commit(tmp_path, False)
command_1 = AtomicCmd(("echo", "Hello, World!"), OUT_STDOUT=AtomicCmd.PIPE)
command_2 = AtomicCmd(("gzip", ),
IN_STDIN=command_1,
OUT_STDOUT=os.path.join(destination, "foo.gz"))
command_1.run(tmp_path)
command_2.run(tmp_path)
assert command_1.join() == [0]
assert command_2.join() == [0]
command_1.commit(tmp_path)
command_2.commit(tmp_path)
assert set(os.listdir(destination)) == set(("foo.gz", ))
assert set(os.listdir(tmp_path)) == set()
def test_atomiccmd__terminate(tmp_path):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(tmp_path)
with patch("os.killpg", wraps=os.killpg) as os_killpg:
cmd.terminate()
assert cmd.join() == ["SIGTERM"]
assert os_killpg.mock_calls == [call(cmd._proc.pid, signal.SIGTERM)]
def test_atomiccmd__commit_with_pipes(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
command_1 = AtomicCmd(("echo", "Hello, World!"), OUT_STDOUT=AtomicCmd.PIPE)
command_2 = AtomicCmd(("gzip", ),
IN_STDIN=command_1,
OUT_STDOUT=os.path.join(destination, "foo.gz"))
command_1.run(temp_folder)
command_2.run(temp_folder)
assert_equal(command_1.join(), [0])
assert_equal(command_2.join(), [0])
command_1.commit(temp_folder)
command_2.commit(temp_folder)
assert_equal(set(os.listdir(destination)), set(("foo.gz", )))
assert_equal(set(os.listdir(temp_folder)), set())
def test_pformat__simple__done__set_cwd(temp_folder):
cmd = AtomicCmd("true", set_cwd=True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd), ("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = 'pipe_true_{id}.stdout'\n"
" STDERR* = 'pipe_true_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
def test_pformat__simple__done__set_cwd(tmp_path):
cmd = AtomicCmd("true", set_cwd=True)
cmd.run(tmp_path)
assert cmd.join() == [0]
assert pformat(cmd) == ("Command = true\n"
"Status = Exited with return-code 0\n"
"STDOUT* = 'pipe_true_{id}.stdout'\n"
"STDERR* = 'pipe_true_{id}.stderr'\n"
"CWD = '{temp_dir}'").format(id=id(cmd),
temp_dir=tmp_path)
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN="infile.fasta",
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, "a\nbc\nd")
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN="infile.fasta",
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, "a\nbc\nd")
def test_atomiccmd__commit_with_pipes(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
command_1 = AtomicCmd(("echo", "Hello, World!"),
OUT_STDOUT=AtomicCmd.PIPE)
command_2 = AtomicCmd(("gzip",),
IN_STDIN=command_1,
OUT_STDOUT=os.path.join(destination, "foo.gz"))
command_1.run(temp_folder)
command_2.run(temp_folder)
assert_equal(command_1.join(), [0])
assert_equal(command_2.join(), [0])
command_1.commit(temp_folder)
command_2.commit(temp_folder)
assert_equal(set(os.listdir(destination)), set(("foo.gz",)))
assert_equal(set(os.listdir(temp_folder)), set())
def test_atomiccmd__pipes_stdin(temp_folder):
fname = test_file("fasta_file.fasta")
cmd = AtomicCmd("cat",
IN_STDIN=fname,
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__pipes_stdin__temp_file(tmp_path):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN="infile.fasta",
OUT_STDOUT="result.txt")
assert cmd.input_files == frozenset()
(tmp_path / "infile.fasta").write_text("a\nbc\nd")
cmd.run(tmp_path)
assert cmd.join() == [0]
result = (tmp_path / "result.txt").read_text()
assert result == "a\nbc\nd"
def test_atomiccmd__pipes_stdin(tmp_path):
fname = tmp_path / "input.fasta"
fname.write_text(">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
fname = str(fname)
cmd = AtomicCmd("cat", IN_STDIN=fname, OUT_STDOUT="result.txt")
assert cmd.input_files == frozenset([fname])
cmd.run(tmp_path)
assert cmd.join() == [0]
result = (tmp_path / "result.txt").read_text()
assert result == ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n"
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def _test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def _test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_pformat__simple__done(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd),
("Command = true\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
assert_equal(paleomix.atomiccmd.command._PROCS, set())
cmd = AtomicCmd("ls")
cmd.run(temp_folder)
ref = iter(paleomix.atomiccmd.command._PROCS).next()
assert ref
assert_equal(ref(), cmd._proc)
assert_equal(cmd.join(), [0])
cmd = func(cmd, temp_folder)
assert ref not in paleomix.atomiccmd.command._PROCS
def test_pformat__simple__killed(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Terminated with signal SIGTERM\n"
" STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd):
cwd = os.getcwd()
cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(cwd, os.getcwd())
expected = temp_folder if set_cwd else cwd
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
def test_commandnode_teardown__missing_optional_files(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
TEMP_OUT_BAR="bar.txt",
OUT_STDOUT=os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
node._teardown(None, temp_folder)
assert_equal(os.listdir(temp_folder), [])
assert_equal(os.listdir(destination), ["foo.txt"])
def test_pformat__simple__temp_root_in_arguments(temp_folder):
cmd = AtomicCmd(("echo", "${TEMP_DIR}"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd),
("Command = echo '{temp_dir}'\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_echo_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_echo_{id}.stderr'\n"
"CWD = '{cwd}'")
.format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def test_pformat__simple__terminated_by_pipeline(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Automatically terminated by PALEOMIX\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def test_pformat__simple__killed_by_signal(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
# pylint: disable=protected-access
os.killpg(cmd._proc.pid, signal.SIGTERM)
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Terminated with signal SIGTERM\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def test_commandnode_teardown(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
OUT_STDOUT=os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
assert os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert not os.path.exists(os.path.join(destination, "foo.txt"))
node._teardown(None, temp_folder)
assert not os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert os.path.exists(os.path.join(destination, "foo.txt"))
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
# pylint: disable=protected-access
cmd._proc.wait()
assert_equal(pformat(cmd),
("Command = true\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
assert_equal(cmd.join(), [0])
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected_files = []
for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
if tmpl is not None:
fname = tmpl.format(id(cmd))
result = get_file_contents(os.path.join(temp_folder, fname))
assert_equal(result, text)
expected_files.append(fname)
assert_equal(set(os.listdir(temp_folder)), set(expected_files))
def _setup_for_commit(temp_folder, create_cmd=True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd