def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def test_atomiccmd__commit_before_join(temp_folder):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.join()
def test_atomiccmd__commit_before_join(temp_folder):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.join()
def _do_test_atomiccmd__stdout(temp_folder, kwargs):
cmd = AtomicCmd(("echo", "-n", "foo"), **kwargs)
assert cmd.stdout is None
cmd.run(temp_folder)
assert isinstance(cmd.stdout, types.FileType), cmd.stdout
assert_equal(cmd.stdout.read(), "foo")
cmd.join()
cmd.commit(temp_folder)
assert cmd.stdout is None
def _do_test_atomiccmd__stdout(temp_folder, kwargs):
cmd = AtomicCmd(("echo", "-n", "foo"), **kwargs)
assert cmd.stdout is None
cmd.run(temp_folder)
assert isinstance(cmd.stdout, types.FileType), cmd.stdout
assert_equal(cmd.stdout.read(), "foo")
cmd.join()
cmd.commit(temp_folder)
assert cmd.stdout is None
def test_atomiccmd__commit_missing_files(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"),
OUT_BAR=os.path.join(destination, "4567"))
cmd.run(temp_folder)
cmd.join()
before = set(os.listdir(temp_folder))
assert_raises(CmdError, cmd.commit, temp_folder)
assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__commit_missing_files(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO = os.path.join(destination, "1234"),
OUT_BAR = os.path.join(destination, "4567"))
cmd.run(temp_folder)
cmd.join()
before = set(os.listdir(temp_folder))
assert_raises(CmdError, cmd.commit, temp_folder)
assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def test_pformat__simple__running__set_cwd(temp_folder):
cmd = AtomicCmd(("sleep", "10"), set_cwd = True)
cmd.run(temp_folder)
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Running ...\n"
" STDOUT* = 'pipe_sleep_{id}.stdout'\n"
" STDERR* = 'pipe_sleep_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id = id(cmd),
temp_dir = temp_folder))
cmd.terminate()
cmd.join()
def test_pformat__simple__running__set_cwd(temp_folder):
cmd = AtomicCmd(("sleep", "10"), set_cwd=True)
cmd.run(temp_folder)
assert_equal(pformat(cmd),
("<Command = ['sleep', '10']\n"
" Status = Running ...\n"
" STDOUT* = 'pipe_sleep_{id}.stdout'\n"
" STDERR* = 'pipe_sleep_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
cmd.terminate()
cmd.join()
def test_atomiccmd__piping_is_only_allowed_once(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"],
OUT_STDOUT=AtomicCmd.PIPE)
cmd_2a = AtomicCmd(["grep", "foo"],
IN_STDIN=cmd_1)
cmd_2b = AtomicCmd(["grep", "bar"],
IN_STDIN=cmd_1)
cmd_1.run(temp_folder)
cmd_2a.run(temp_folder)
assert_raises(CmdError, cmd_2b.run, temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2a.join(), [0])
assert_equal(cmd_2b.join(), [None])
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"],
TEMP_OUT_STDOUT = AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"],
TEMP_IN_STDIN = cmd_1,
OUT_STDOUT = "piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def test_atomiccmd__paths__key(temp_folder):
cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"),
OUT_STDOUT=AtomicCmd.PIPE)
cmd.run(temp_folder)
path = cmd._proc.stdout.read()
assert os.path.samefile(temp_folder, path), (temp_folder, path)
assert_equal(cmd.join(), [0])
def test_atomiccmd__paths_non_str(temp_folder):
cmd = AtomicCmd(("touch", 1234),
OUT_FOO = "1234",
set_cwd = True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "1234"))
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def test_atomiccmd__terminate_race_condition(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__terminate_race_condition(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"),
TEMP_OUT_STDOUT = "bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd))))
result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd))))
assert_equal(result_out, "STDOUT!")
assert_equal(result_err, "STDERR!")
def test_atomiccmd__pipes_stdin(temp_folder):
fname = os.path.join("tests", "data", "fasta_file.fasta")
cmd = AtomicCmd("cat", IN_STDIN=fname, OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result,
">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_pformat__simple__done__set_cwd(temp_folder):
cmd = AtomicCmd("true", set_cwd = True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd), ("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = 'pipe_true_{id}.stdout'\n"
" STDERR* = 'pipe_true_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id = id(cmd),
temp_dir = temp_folder))
def test_atomiccmd__pipes_stdin(temp_folder):
fname = os.path.join("tests", "data", "fasta_file.fasta")
cmd = AtomicCmd("cat",
IN_STDIN = fname,
OUT_STDOUT = "result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN = "infile.fasta",
OUT_STDOUT = "result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "infile.fasta"))
assert_equal(result, "a\nbc\nd")
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN="infile.fasta",
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "infile.fasta"))
assert_equal(result, "a\nbc\nd")
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT = "result.txt",
set_cwd = set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd = False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT = os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO = "bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_pformat__simple__done__set_cwd(temp_folder):
cmd = AtomicCmd("true", set_cwd=True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd),
("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = 'pipe_true_{id}.stdout'\n"
" STDERR* = 'pipe_true_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def test_commandnode_teardown__missing_optional_files(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
TEMP_OUT_BAR = "bar.txt",
OUT_STDOUT = os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
node._teardown(None, temp_folder)
assert_equal(os.listdir(temp_folder), [])
assert_equal(os.listdir(destination), ["foo.txt"])
def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
assert_equal(pypeline.atomiccmd.command._PROCS, set())
cmd = AtomicCmd("ls")
cmd.run(temp_folder)
ref = iter(pypeline.atomiccmd.command._PROCS).next()
assert ref
assert_equal(ref(), cmd._proc)
assert_equal(cmd.join(), [0])
cmd = func(cmd, temp_folder)
assert ref not in pypeline.atomiccmd.command._PROCS
def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
assert_equal(pypeline.atomiccmd.command._PROCS, set())
cmd = AtomicCmd("ls")
cmd.run(temp_folder)
ref = iter(pypeline.atomiccmd.command._PROCS).next()
assert ref
assert_equal(ref(), cmd._proc)
assert_equal(cmd.join(), [0])
cmd = func(cmd, temp_folder)
assert ref not in pypeline.atomiccmd.command._PROCS
def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd):
cwd = os.getcwd()
cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"),
TEMP_OUT_STDOUT = "result.txt",
set_cwd = set_cwd)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(cwd, os.getcwd())
expected = temp_folder if set_cwd else cwd
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
cmd._proc.wait() # pylint: disable=W0212
assert_equal(pformat(cmd), ("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id = id(cmd),
cwd = os.getcwd(),
temp_dir = temp_folder))
assert_equal(cmd.join(), [0])
def test_pformat__simple__killed(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Terminated with signal SIGTERM\n"
" STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id = id(cmd),
temp_dir = temp_folder,
cwd = os.getcwd()))
def test_pformat__simple__killed(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd),
("<Command = ['sleep', '10']\n"
" Status = Terminated with signal SIGTERM\n"
" STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def test_commandnode_teardown(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
OUT_STDOUT = os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
assert os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert not os.path.exists(os.path.join(destination, "foo.txt"))
node._teardown(None, temp_folder)
assert not os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert os.path.exists(os.path.join(destination, "foo.txt"))
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
cmd._proc.wait() # pylint: disable=W0212
assert_equal(pformat(cmd),
("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
assert_equal(cmd.join(), [0])
def _setup_for_commit(temp_folder, create_cmd = True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO = os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd
def test_commandnode_teardown__extra_files_in_temp(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
OUT_STDOUT = os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
set_file_contents(os.path.join(temp_folder, "bar.txt"), "1 2 3")
temp_files_before = set(os.listdir(temp_folder))
dest_files_before = set(os.listdir(destination))
assert_raises(CmdNodeError, node._teardown, None, temp_folder)
assert_equal(temp_files_before, set(os.listdir(temp_folder)))
assert_equal(dest_files_before, set(os.listdir(destination)))
def _setup_for_commit(temp_folder, create_cmd=True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd
def _do_test_atomiccmd__terminate(temp_folder, raise_on_terminate):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
killpg_was_called = []
def _wrap_killpg(pid, sig):
assert_equal(pid, cmd._proc.pid)
assert_equal(sig, signal.SIGTERM)
killpg_was_called.append(True)
if raise_on_terminate:
raise OSError("KABOOM!")
with Monkeypatch("os.killpg", _wrap_killpg):
cmd.terminate()
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert killpg_was_called
def test_atomiccmd__commit_while_running(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.terminate()
cmd.join()
def test_atomiccmd__terminate_after_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__terminate_sigterm(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
def _do_test_atomiccmd__stdout_no_pipe(temp_folder, kwargs):
cmd = AtomicCmd(("echo", "foo"), **kwargs)
assert cmd.stdout is None
cmd.run(temp_folder)
assert cmd.stdout is None
cmd.join()
def test_atomiccmd__terminate_sigkill(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd._proc.kill()
assert_equal(cmd.join(), ["SIGKILL"])
def test_atomiccmd__terminate_sigterm(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
def test_atomiccmd__commit_while_running(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.terminate()
cmd.join()
def test_atomiccmd__terminate_sigkill(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd._proc.kill()
assert_equal(cmd.join(), ["SIGKILL"])
