def test_atomiccmd__terminate_race_condition(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) while cmd._proc.poll() is None: pass cmd.terminate() assert_equal(cmd.join(), [0])
def test_atomiccmd__commit_temp_only(temp_folder): cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "bar.txt")) cmd.commit(temp_folder) assert_equal(os.listdir(temp_folder), [])
def test_atomiccmd__ready(temp_folder): cmd = AtomicCmd("ls") assert_equal(cmd.join(), [None]) assert not cmd.ready() cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert cmd.ready()
def test_atomiccmd__commit_before_join(temp_folder): cmd = AtomicCmd(("sleep", "0.1")) cmd.run(temp_folder) while cmd._proc.poll() is None: pass assert_raises(CmdError, cmd.commit, temp_folder) cmd.join()
def test_atomiccmd__paths_non_str(temp_folder): cmd = AtomicCmd(("touch", 1234), OUT_FOO = "1234", set_cwd = True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "1234"))
def test_atomiccmd__paths__key(temp_folder): cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"), OUT_STDOUT=AtomicCmd.PIPE) cmd.run(temp_folder) path = cmd._proc.stdout.read() assert os.path.samefile(temp_folder, path), (temp_folder, path) assert_equal(cmd.join(), [0])
def test_atomiccmd__commit_temp_only(temp_folder): cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT = "bar.txt") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "bar.txt")) cmd.commit(temp_folder) assert_equal(os.listdir(temp_folder), [])
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs): cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd)))) result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd)))) assert_equal(result_out, "STDOUT!") assert_equal(result_err, "STDERR!")
def test_atomiccmd__pipes_stdin(temp_folder): fname = os.path.join("tests", "data", "fasta_file.fasta") cmd = AtomicCmd("cat", IN_STDIN=fname, OUT_STDOUT="result.txt") assert_equal(cmd.input_files, frozenset([fname])) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def _do_test_atomiccmd__stdout(temp_folder, kwargs): cmd = AtomicCmd(("echo", "-n", "foo"), **kwargs) assert cmd.stdout is None cmd.run(temp_folder) assert isinstance(cmd.stdout, types.FileType), cmd.stdout assert_equal(cmd.stdout.read(), "foo") cmd.join() cmd.commit(temp_folder) assert cmd.stdout is None
def test_atomiccmd__pipes_stdin__temp_file(temp_folder): cmd = AtomicCmd("cat", TEMP_IN_STDIN="infile.fasta", OUT_STDOUT="result.txt") assert_equal(cmd.input_files, frozenset()) set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "infile.fasta")) assert_equal(result, "a\nbc\nd")
def test_atomiccmd__commit_missing_files(temp_folder): destination, temp_folder = _setup_for_commit(temp_folder, False) cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO=os.path.join(destination, "1234"), OUT_BAR=os.path.join(destination, "4567")) cmd.run(temp_folder) cmd.join() before = set(os.listdir(temp_folder)) assert_raises(CmdError, cmd.commit, temp_folder) assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__commit_missing_files(temp_folder): destination, temp_folder = _setup_for_commit(temp_folder, False) cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO = os.path.join(destination, "1234"), OUT_BAR = os.path.join(destination, "4567")) cmd.run(temp_folder) cmd.join() before = set(os.listdir(temp_folder)) assert_raises(CmdError, cmd.commit, temp_folder) assert_equal(before, set(os.listdir(temp_folder)))
def test_pformat__simple__done__set_cwd(temp_folder): cmd = AtomicCmd("true", set_cwd = True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = 'pipe_true_{id}.stdout'\n" " STDERR* = 'pipe_true_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id = id(cmd), temp_dir = temp_folder))
def test_atomiccmd__pipes_stdin(temp_folder): fname = os.path.join("tests", "data", "fasta_file.fasta") cmd = AtomicCmd("cat", IN_STDIN = fname, OUT_STDOUT = "result.txt") assert_equal(cmd.input_files, frozenset([fname])) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__pipes_stdin__temp_file(temp_folder): cmd = AtomicCmd("cat", TEMP_IN_STDIN = "infile.fasta", OUT_STDOUT = "result.txt") assert_equal(cmd.input_files, frozenset()) set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "infile.fasta")) assert_equal(result, "a\nbc\nd")
def test_atomiccmd__commit_temp_out(temp_folder): dest, temp = _setup_for_commit(temp_folder, create_cmd = False) cmd = AtomicCmd(("echo", "foo"), OUT_STDOUT = os.path.join(dest, "foo.txt"), TEMP_OUT_FOO = "bar.txt") cmd.run(temp) assert_equal(cmd.join(), [0]) set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3") cmd.commit(temp) assert_equal(os.listdir(temp), []) assert_equal(os.listdir(dest), ["foo.txt"])
def test_atomiccmd__commit_temp_out(temp_folder): dest, temp = _setup_for_commit(temp_folder, create_cmd=False) cmd = AtomicCmd(("echo", "foo"), OUT_STDOUT=os.path.join(dest, "foo.txt"), TEMP_OUT_FOO="bar.txt") cmd.run(temp) assert_equal(cmd.join(), [0]) set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3") cmd.commit(temp) assert_equal(os.listdir(temp), []) assert_equal(os.listdir(dest), ["foo.txt"])
def test_atomiccmd__piping_temp(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE) assert_equal(cmd_1.output_files, frozenset()) cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt") assert_equal(cmd_2.input_files, frozenset()) cmd_1.run(temp_folder) cmd_2.run(temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "piped.txt")) assert_equal(result, "#@!$^")
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs): cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())), TEMP_OUT_STDOUT = "result.txt", set_cwd = set_cwd, **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) expected = os.path.join("" if set_cwd else temp_folder, "test_file") result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(os.path.abspath(expected), os.path.abspath(result))
def test_pformat__simple__done__set_cwd(temp_folder): cmd = AtomicCmd("true", set_cwd=True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = 'pipe_true_{id}.stdout'\n" " STDERR* = 'pipe_true_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id=id(cmd), temp_dir=temp_folder))
def test_pformat__simple__running__set_cwd(temp_folder): cmd = AtomicCmd(("sleep", "10"), set_cwd = True) cmd.run(temp_folder) assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n" " Status = Running ...\n" " STDOUT* = 'pipe_sleep_{id}.stdout'\n" " STDERR* = 'pipe_sleep_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id = id(cmd), temp_dir = temp_folder)) cmd.terminate() cmd.join()
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs): cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())), TEMP_OUT_STDOUT="result.txt", set_cwd=set_cwd, **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) expected = os.path.join("" if set_cwd else temp_folder, "test_file") result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(os.path.abspath(expected), os.path.abspath(result))
def test_pformat__simple__done__before_join(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) cmd._proc.wait() # pylint: disable=W0212 assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n" " CWD = '{cwd}'>").format(id = id(cmd), cwd = os.getcwd(), temp_dir = temp_folder)) assert_equal(cmd.join(), [0])
def test_commandnode_teardown__missing_optional_files(temp_folder): destination, temp_folder = _setup_temp_folders(temp_folder) cmd = AtomicCmd(("echo", "-n", "1 2 3"), TEMP_OUT_BAR = "bar.txt", OUT_STDOUT = os.path.join(destination, "foo.txt")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) node = CommandNode(cmd) node._teardown(None, temp_folder) assert_equal(os.listdir(temp_folder), []) assert_equal(os.listdir(destination), ["foo.txt"])
def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd): cwd = os.getcwd() cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"), TEMP_OUT_STDOUT = "result.txt", set_cwd = set_cwd) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert_equal(cwd, os.getcwd()) expected = temp_folder if set_cwd else cwd result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
def _do_test_atomiccmd__cleanup_proc(temp_folder, func): assert_equal(pypeline.atomiccmd.command._PROCS, set()) cmd = AtomicCmd("ls") cmd.run(temp_folder) ref = iter(pypeline.atomiccmd.command._PROCS).next() assert ref assert_equal(ref(), cmd._proc) assert_equal(cmd.join(), [0]) cmd = func(cmd, temp_folder) assert ref not in pypeline.atomiccmd.command._PROCS
def test_pformat__simple__killed(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"]) assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n" " Status = Terminated with signal SIGTERM\n" " STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n" " CWD = '{cwd}'>").format(id = id(cmd), temp_dir = temp_folder, cwd = os.getcwd()))
def test_pformat__simple__running__set_cwd(temp_folder): cmd = AtomicCmd(("sleep", "10"), set_cwd=True) cmd.run(temp_folder) assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n" " Status = Running ...\n" " STDOUT* = 'pipe_sleep_{id}.stdout'\n" " STDERR* = 'pipe_sleep_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id=id(cmd), temp_dir=temp_folder)) cmd.terminate() cmd.join()
def test_commandnode_teardown(temp_folder): destination, temp_folder = _setup_temp_folders(temp_folder) cmd = AtomicCmd(("echo", "-n", "1 2 3"), OUT_STDOUT = os.path.join(destination, "foo.txt")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) node = CommandNode(cmd) assert os.path.exists(os.path.join(temp_folder, "foo.txt")) assert not os.path.exists(os.path.join(destination, "foo.txt")) node._teardown(None, temp_folder) assert not os.path.exists(os.path.join(temp_folder, "foo.txt")) assert os.path.exists(os.path.join(destination, "foo.txt"))
def test_pformat__simple__killed(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"]) assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n" " Status = Terminated with signal SIGTERM\n" " STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n" " CWD = '{cwd}'>").format(id=id(cmd), temp_dir=temp_folder, cwd=os.getcwd()))
def test_atomiccmd__piping_is_only_allowed_once(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"], OUT_STDOUT=AtomicCmd.PIPE) cmd_2a = AtomicCmd(["grep", "foo"], IN_STDIN=cmd_1) cmd_2b = AtomicCmd(["grep", "bar"], IN_STDIN=cmd_1) cmd_1.run(temp_folder) cmd_2a.run(temp_folder) assert_raises(CmdError, cmd_2b.run, temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2a.join(), [0]) assert_equal(cmd_2b.join(), [None])
def test_pformat__simple__done__before_join(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) cmd._proc.wait() # pylint: disable=W0212 assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n" " CWD = '{cwd}'>").format(id=id(cmd), cwd=os.getcwd(), temp_dir=temp_folder)) assert_equal(cmd.join(), [0])
def test_atomiccmd__piping_temp(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT = AtomicCmd.PIPE) assert_equal(cmd_1.output_files, frozenset()) cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN = cmd_1, OUT_STDOUT = "piped.txt") assert_equal(cmd_2.input_files, frozenset()) cmd_1.run(temp_folder) cmd_2.run(temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "piped.txt")) assert_equal(result, "#@!$^")
def test_commandnode_teardown__extra_files_in_temp(temp_folder): destination, temp_folder = _setup_temp_folders(temp_folder) cmd = AtomicCmd(("echo", "-n", "1 2 3"), OUT_STDOUT = os.path.join(destination, "foo.txt")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) node = CommandNode(cmd) set_file_contents(os.path.join(temp_folder, "bar.txt"), "1 2 3") temp_files_before = set(os.listdir(temp_folder)) dest_files_before = set(os.listdir(destination)) assert_raises(CmdNodeError, node._teardown, None, temp_folder) assert_equal(temp_files_before, set(os.listdir(temp_folder))) assert_equal(dest_files_before, set(os.listdir(destination)))
def _setup_for_commit(temp_folder, create_cmd = True): destination = os.path.join(temp_folder, "out") temp_folder = os.path.join(temp_folder, "tmp") os.makedirs(destination) os.makedirs(temp_folder) if not create_cmd: return destination, temp_folder cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO = os.path.join(destination, "1234")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) return destination, temp_folder, cmd
def _setup_for_commit(temp_folder, create_cmd=True): destination = os.path.join(temp_folder, "out") temp_folder = os.path.join(temp_folder, "tmp") os.makedirs(destination) os.makedirs(temp_folder) if not create_cmd: return destination, temp_folder cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO=os.path.join(destination, "1234")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) return destination, temp_folder, cmd
def _do_test_atomiccmd__terminate(temp_folder, raise_on_terminate): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) killpg_was_called = [] def _wrap_killpg(pid, sig): assert_equal(pid, cmd._proc.pid) assert_equal(sig, signal.SIGTERM) killpg_was_called.append(True) if raise_on_terminate: raise OSError("KABOOM!") with Monkeypatch("os.killpg", _wrap_killpg): cmd.terminate() cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"]) assert killpg_was_called
def test_atomiccmd__commit_while_running(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) assert_raises(CmdError, cmd.commit, temp_folder) cmd.terminate() cmd.join()
def test_atomiccmd__terminate_after_join(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) cmd.terminate() assert_equal(cmd.join(), [0])
def _do_test_atomiccmd__stdout_no_pipe(temp_folder, kwargs): cmd = AtomicCmd(("echo", "foo"), **kwargs) assert cmd.stdout is None cmd.run(temp_folder) assert cmd.stdout is None cmd.join()
def test_atomiccmd__terminate_sigterm(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"])
def test_atomiccmd__terminate_sigkill(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd._proc.kill() assert_equal(cmd.join(), ["SIGKILL"])