def test_sequential_commands__abort_on_error_1(temp_folder): cmd_1 = AtomicCmd("false") cmd_2 = AtomicCmd(("sleep", 10)) cmd_3 = AtomicCmd(("sleep", 10)) cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(temp_folder) assert_equal(cmds.join(), [1, None, None])
def _do_test(cls): cmd_mock_1 = AtomicCmd(("true", ), CHECK_A=id, EXEC_1="false", IN_1="/foo/bar/in_1.file", IN_2="/foo/bar/in_2.file", OUT_1="/bar/foo/out", TEMP_OUT_1="out.log", AUX_A="/aux/fA", AUX_B="/aux/fB") cmd_mock_2 = AtomicCmd(("false", ), CHECK_A=list, EXEC_1="echo", EXEC_2="java", IN_1="/foo/bar/in.file", OUT_1="out.txt") obj = cls([cmd_mock_1, cmd_mock_2]) assert_equal(obj.executables, cmd_mock_1.executables | cmd_mock_2.executables) assert_equal(obj.requirements, cmd_mock_1.requirements | cmd_mock_2.requirements) assert_equal(obj.input_files, cmd_mock_1.input_files | cmd_mock_2.input_files) assert_equal(obj.output_files, cmd_mock_1.output_files | cmd_mock_2.output_files) assert_equal(obj.auxiliary_files, cmd_mock_1.auxiliary_files | cmd_mock_2.auxiliary_files) assert_equal(obj.expected_temp_files, frozenset(["out", "out.txt"])) assert_equal( obj.optional_temp_files, cmd_mock_1.optional_temp_files | cmd_mock_2.optional_temp_files)
def __init__(self, config, reference, input_files, output_file, dependencies): cat_cmds, cat_obj = concatenate_input_bams(config, input_files) cmd_map = AtomicCmd([ "mapDamage", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d", "%(TEMP_DIR)s", "-r", reference ], IN_STDIN=cat_obj, CHECK_VERSION=MAPDAMAGE_VERSION) train_cmds = ParallelCmds(cat_cmds + [cmd_map]) cat_cmds, cat_obj = concatenate_input_bams(config, input_files) cmd_scale = AtomicCmd([ "mapDamage", "--rescale-only", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d", "%(TEMP_DIR)s", "-r", reference, "--rescale-out", "%(OUT_BAM)s" ], IN_STDIN=cat_obj, OUT_BAM=output_file, CHECK_VERSION=MAPDAMAGE_VERSION) rescale_cmds = ParallelCmds(cat_cmds + [cmd_scale]) description = "<mapDamageRescale: %i file(s) -> '%s'>" % ( len(input_files), output_file) CommandNode.__init__(self, command=SequentialCmds([train_cmds, rescale_cmds]), description=description, dependencies=dependencies)
def test_atomiccmd__commit_before_join(temp_folder): cmd = AtomicCmd(("sleep", "0.1")) cmd.run(temp_folder) while cmd._proc.poll() is None: pass assert_raises(CmdError, cmd.commit, temp_folder) cmd.join()
def test_atomiccmd__paths_non_str(temp_folder): cmd = AtomicCmd(("touch", 1234), OUT_FOO = "1234", set_cwd = True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "1234"))
def test_atomiccmd__paths__key(temp_folder): cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"), OUT_STDOUT=AtomicCmd.PIPE) cmd.run(temp_folder) path = cmd._proc.stdout.read() assert os.path.samefile(temp_folder, path), (temp_folder, path) assert_equal(cmd.join(), [0])
def _do_test_parallel_commands__ready_two(first, second, result): cmd_mock_1 = flexmock(AtomicCmd(["ls"])) cmd_mock_1.should_receive('ready').and_return(first).at_least.once cmd_mock_2 = flexmock(AtomicCmd(["ls"])) cmd_mock_2.should_receive('ready').and_return(second) cmds = ParallelCmds([cmd_mock_1, cmd_mock_2]) assert_equal(cmds.ready(), result)
def test_sequential_commands__abort_on_error_3(temp_folder): cmd_1 = AtomicCmd("true") cmd_2 = AtomicCmd("true") cmd_3 = AtomicCmd("false") cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(temp_folder) assert_equal(cmds.join(), [0, 0, 1])
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs): cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd)))) result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd)))) assert_equal(result_out, "STDOUT!") assert_equal(result_err, "STDERR!")
def test_pformat__atomiccmd__simple_with_stdin__cmd(): cmd_1 = AtomicCmd("gzip", OUT_STDOUT=AtomicCmd.PIPE) cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1) assert_equal(pformat(cmd_2), ("<Command = ['gzip']\n" " STDIN = <PIPE>\n" " STDOUT* = '${TEMP_DIR}/pipe_gzip_%i.stdout'\n" " STDERR* = '${TEMP_DIR}/pipe_gzip_%i.stderr'>") % (id(cmd_2), id(cmd_2)))
def test_atomiccmd__pipes_stdin(temp_folder): fname = os.path.join("tests", "data", "fasta_file.fasta") cmd = AtomicCmd("cat", IN_STDIN=fname, OUT_STDOUT="result.txt") assert_equal(cmd.input_files, frozenset([fname])) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__commit_missing_files(temp_folder): destination, temp_folder = _setup_for_commit(temp_folder, False) cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO = os.path.join(destination, "1234"), OUT_BAR = os.path.join(destination, "4567")) cmd.run(temp_folder) cmd.join() before = set(os.listdir(temp_folder)) assert_raises(CmdError, cmd.commit, temp_folder) assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__pipes_stdin__temp_file(temp_folder): cmd = AtomicCmd("cat", TEMP_IN_STDIN = "infile.fasta", OUT_STDOUT = "result.txt") assert_equal(cmd.input_files, frozenset()) set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "infile.fasta")) assert_equal(result, "a\nbc\nd")
def test_atomiccmd__pipes_stdin(temp_folder): fname = os.path.join("tests", "data", "fasta_file.fasta") cmd = AtomicCmd("cat", IN_STDIN = fname, OUT_STDOUT = "result.txt") assert_equal(cmd.input_files, frozenset([fname])) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_pformat__simple__done__set_cwd(temp_folder): cmd = AtomicCmd("true", set_cwd = True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = 'pipe_true_{id}.stdout'\n" " STDERR* = 'pipe_true_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id = id(cmd), temp_dir = temp_folder))
def test_atomiccmd__pipes_stdin__temp_file(temp_folder): cmd = AtomicCmd("cat", TEMP_IN_STDIN="infile.fasta", OUT_STDOUT="result.txt") assert_equal(cmd.input_files, frozenset()) set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "infile.fasta")) assert_equal(result, "a\nbc\nd")
def test_pformat__simple__done__set_cwd(temp_folder): cmd = AtomicCmd("true", set_cwd=True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = 'pipe_true_{id}.stdout'\n" " STDERR* = 'pipe_true_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id=id(cmd), temp_dir=temp_folder))
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs): cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())), TEMP_OUT_STDOUT="result.txt", set_cwd=set_cwd, **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) expected = os.path.join("" if set_cwd else temp_folder, "test_file") result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(os.path.abspath(expected), os.path.abspath(result))
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs): cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())), TEMP_OUT_STDOUT = "result.txt", set_cwd = set_cwd, **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) expected = os.path.join("" if set_cwd else temp_folder, "test_file") result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert_equal(os.path.abspath(expected), os.path.abspath(result))
def _setup_mocks_for_failure(*do_mocks): results = [] for do_mock in do_mocks: if do_mock: mock = flexmock(AtomicCmd(("sleep", 10))) mock.should_receive('terminate') mock.should_receive('join').and_return(['SIGTERM']) else: mock = AtomicCmd("false") results.append(mock) return results
def test_atomiccmd__piping_temp(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE) assert_equal(cmd_1.output_files, frozenset()) cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt") assert_equal(cmd_2.input_files, frozenset()) cmd_1.run(temp_folder) cmd_2.run(temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "piped.txt")) assert_equal(result, "#@!$^")
def _do_test_atomiccmd__cleanup_proc(temp_folder, func): assert_equal(pypeline.atomiccmd.command._PROCS, set()) cmd = AtomicCmd("ls") cmd.run(temp_folder) ref = iter(pypeline.atomiccmd.command._PROCS).next() assert ref assert_equal(ref(), cmd._proc) assert_equal(cmd.join(), [0]) cmd = func(cmd, temp_folder) assert ref not in pypeline.atomiccmd.command._PROCS
def test_pformat__simple__done__before_join(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) cmd._proc.wait() # pylint: disable=W0212 assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n" " CWD = '{cwd}'>").format(id = id(cmd), cwd = os.getcwd(), temp_dir = temp_folder)) assert_equal(cmd.join(), [0])
def test_commandnode_teardown__missing_optional_files(temp_folder): destination, temp_folder = _setup_temp_folders(temp_folder) cmd = AtomicCmd(("echo", "-n", "1 2 3"), TEMP_OUT_BAR = "bar.txt", OUT_STDOUT = os.path.join(destination, "foo.txt")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) node = CommandNode(cmd) node._teardown(None, temp_folder) assert_equal(os.listdir(temp_folder), []) assert_equal(os.listdir(destination), ["foo.txt"])
def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd): cwd = os.getcwd() cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"), TEMP_OUT_STDOUT = "result.txt", set_cwd = set_cwd) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert_equal(cwd, os.getcwd()) expected = temp_folder if set_cwd else cwd result = get_file_contents(os.path.join(temp_folder, "result.txt")) assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
def _do_test_atomiccmd__cleanup_proc(temp_folder, func): assert_equal(pypeline.atomiccmd.command._PROCS, set()) cmd = AtomicCmd("ls") cmd.run(temp_folder) ref = iter(pypeline.atomiccmd.command._PROCS).next() assert ref assert_equal(ref(), cmd._proc) assert_equal(cmd.join(), [0]) cmd = func(cmd, temp_folder) assert ref not in pypeline.atomiccmd.command._PROCS
def test_pformat__simple__done__before_join(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) cmd._proc.wait() # pylint: disable=W0212 assert_equal(pformat(cmd), ("<Command = ['true']\n" " Status = Exited with return-code 0\n" " STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n" " CWD = '{cwd}'>").format(id=id(cmd), cwd=os.getcwd(), temp_dir=temp_folder)) assert_equal(cmd.join(), [0])
def test_commandnode_teardown(temp_folder): destination, temp_folder = _setup_temp_folders(temp_folder) cmd = AtomicCmd(("echo", "-n", "1 2 3"), OUT_STDOUT = os.path.join(destination, "foo.txt")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) node = CommandNode(cmd) assert os.path.exists(os.path.join(temp_folder, "foo.txt")) assert not os.path.exists(os.path.join(destination, "foo.txt")) node._teardown(None, temp_folder) assert not os.path.exists(os.path.join(temp_folder, "foo.txt")) assert os.path.exists(os.path.join(destination, "foo.txt"))
def test_atomiccmd__piping_is_only_allowed_once(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"], OUT_STDOUT=AtomicCmd.PIPE) cmd_2a = AtomicCmd(["grep", "foo"], IN_STDIN=cmd_1) cmd_2b = AtomicCmd(["grep", "bar"], IN_STDIN=cmd_1) cmd_1.run(temp_folder) cmd_2a.run(temp_folder) assert_raises(CmdError, cmd_2b.run, temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2a.join(), [0]) assert_equal(cmd_2b.join(), [None])
def test_atomiccmd__ready(temp_folder): cmd = AtomicCmd("ls") assert_equal(cmd.join(), [None]) assert not cmd.ready() cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert cmd.ready()
def _do_test_pformat__sets__simple(cls, description): cmd_1 = AtomicCmd(("echo", "foo"), OUT_STDOUT=AtomicCmd.PIPE) cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1) cmd = cls((cmd_1, cmd_2)) assert_equal(pformat(cmd), ("<{description}:\n" " - <00> Command = ['echo', 'foo']\n" " STDOUT = <01>\n" " STDERR* = '${{TEMP_DIR}}/pipe_echo_{cmd_1_id}.stderr'\n" " - <01> Command = ['gzip']\n" " STDIN = <00>\n" " STDOUT* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2_id}.stdout'\n" " STDERR* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2_id}.stderr'>") \ .format(description = description, cmd_1_id = id(cmd_1), cmd_2_id = id(cmd_2)))
def test_commandnode_teardown__extra_files_in_temp(temp_folder): destination, temp_folder = _setup_temp_folders(temp_folder) cmd = AtomicCmd(("echo", "-n", "1 2 3"), OUT_STDOUT = os.path.join(destination, "foo.txt")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) node = CommandNode(cmd) set_file_contents(os.path.join(temp_folder, "bar.txt"), "1 2 3") temp_files_before = set(os.listdir(temp_folder)) dest_files_before = set(os.listdir(destination)) assert_raises(CmdNodeError, node._teardown, None, temp_folder) assert_equal(temp_files_before, set(os.listdir(temp_folder))) assert_equal(dest_files_before, set(os.listdir(destination)))
def _setup_for_commit(temp_folder, create_cmd = True): destination = os.path.join(temp_folder, "out") temp_folder = os.path.join(temp_folder, "tmp") os.makedirs(destination) os.makedirs(temp_folder) if not create_cmd: return destination, temp_folder cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO = os.path.join(destination, "1234")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) return destination, temp_folder, cmd
def _setup_for_commit(temp_folder, create_cmd=True): destination = os.path.join(temp_folder, "out") temp_folder = os.path.join(temp_folder, "tmp") os.makedirs(destination) os.makedirs(temp_folder) if not create_cmd: return destination, temp_folder cmd = AtomicCmd(("touch", "%(OUT_FOO)s"), OUT_FOO=os.path.join(destination, "1234")) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) return destination, temp_folder, cmd
def __init__(self, config, d_make, bedn, mappa, unique, dependencies=()): inbedfile = d_make.bedfiles[bedn] basename, extension = os.path.splitext(os.path.basename(inbedfile)) bname = "{}_MappaOnly{}".format(basename, extension) dest = os.path.join(config.temp_local, bname) d_make.bedfiles[bedn] = dest call1 = [ "python", os.path.join(PREFIX, "intersectmappabed.py"), "%(IN_BED)s", "%(IN_MAPPA)s", str(unique), "%(OUT_DEST)s" ] cmd = AtomicCmd(call1, IN_BED=inbedfile, IN_MAPPA=mappa, OUT_DEST=dest, CHECK_VERSION=PYTHON_VERSION) description = ("<CLEANBEDFILES: '%s' -> '%s', Uniqueness: '%s'>" % (inbedfile, dest, unique)) CommandNode.__init__(self, description=description, command=cmd, dependencies=dependencies)
def __init__(self, infile, outfile, genome, from_start=0, from_end=0, strand_relative=False, dependencies=()): if type(from_start) != type(from_end): raise ValueError( "'from_start' and 'from_end' should be of same type!") call = [ "slopBed", "-i", "%(IN_FILE)s", "-g", "%(IN_GENOME)s", "-l", str(from_start), "-r", str(from_end) ] if strand_relative: call.append("-s") if type(from_start) is float: call.append("-pct") command = AtomicCmd(call, IN_FILE=infile, IN_GENOME=genome, OUT_STDOUT=outfile) description = "<SlopBed: '%s' -> '%s'>" % (infile, outfile) CommandNode.__init__(self, description=description, command=command, dependencies=dependencies)
def __init__(self, config, reference, input_files, output_directory, dependencies): cat_cmds, cat_obj = concatenate_input_bams(config, input_files) cmd_map = AtomicCmd( [ "mapDamage", "--no-stats", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d", "%(TEMP_DIR)s", "-r", reference ], IN_STDIN=cat_obj, OUT_FREQ_3p=os.path.join(output_directory, "3pGtoA_freq.txt"), OUT_FREQ_5p=os.path.join(output_directory, "5pCtoT_freq.txt"), OUT_COMP_USER=os.path.join(output_directory, "dnacomp.txt"), OUT_PLOT_FRAG=os.path.join(output_directory, "Fragmisincorporation_plot.pdf"), OUT_PLOT_LEN=os.path.join(output_directory, "Length_plot.pdf"), OUT_LENGTH=os.path.join(output_directory, "lgdistribution.txt"), OUT_MISINCORP=os.path.join(output_directory, "misincorporation.txt"), OUT_LOG=os.path.join(output_directory, "Runtime_log.txt"), CHECK_VERSION=MAPDAMAGE_VERSION) description = "<mapDamage: %i file(s) -> '%s'>" % (len(input_files), output_directory) CommandNode.__init__(self, command=ParallelCmds(cat_cmds + [cmd_map]), description=description, dependencies=dependencies)
def __init__(self, config, reference, intervals, infiles, outfile, dependencies=()): self._basename = os.path.basename(outfile) infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) command.set_option("-T", "IndelRealigner") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-targetIntervals", "%(IN_INTERVALS)s") command.set_option("-o", "%(OUT_BAMFILE)s") command.set_option("--bam_compression", 0) command.set_option("--disable_bam_indexing") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE=reference, IN_REF_DICT=fileutils.swap_ext(reference, ".dict"), IN_INTERVALS=intervals, OUT_BAMFILE=outfile, CHECK_GATK=_get_gatk_version_check(config)) calmd = AtomicCmd(["samtools", "calmd", "-b", "%(TEMP_IN_BAM)s", "%(IN_REF)s"], TEMP_IN_BAM=self._basename, IN_REF=reference, TEMP_OUT_STDOUT=self._basename + ".calmd", CHECK_VERSION=SAMTOOLS_VERSION) description = "<Indel Realigner (aligning): %s -> %r>" \ % (describe_files(infiles), outfile) CommandNode.__init__(self, description=description, command=ParallelCmds([command.finalize(), calmd]), dependencies=dependencies)
def finalize(self): """Creates an AtomicCmd object based on the AtomicParam object. Once finalized, the AtomicCmdBuilder cannot be modified further.""" if not self._object: self._object = AtomicCmd(self.call, **self.kwargs) return self._object
def test_atomicmcd__exec__reqobj(): reqobj = RequirementObj(call=("echo", "version"), search="version", pprint="{0}", checks=str) cmd = AtomicCmd("true", CHECK_VERSION=reqobj) assert_equal(cmd.requirements, frozenset([reqobj]))
def test_atomiccmd__ready(temp_folder): cmd = AtomicCmd("ls") assert_equal(cmd.join(), [None]) assert not cmd.ready() cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert cmd.ready()
def _do_test_atomiccmd__stdout(temp_folder, kwargs): cmd = AtomicCmd(("echo", "-n", "foo"), **kwargs) assert cmd.stdout is None cmd.run(temp_folder) assert isinstance(cmd.stdout, types.FileType), cmd.stdout assert_equal(cmd.stdout.read(), "foo") cmd.join() cmd.commit(temp_folder) assert cmd.stdout is None
def test_atomiccmd__terminate_race_condition(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) while cmd._proc.poll() is None: pass cmd.terminate() assert_equal(cmd.join(), [0])
def test_atomiccmd__piping_temp(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT = AtomicCmd.PIPE) assert_equal(cmd_1.output_files, frozenset()) cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN = cmd_1, OUT_STDOUT = "piped.txt") assert_equal(cmd_2.input_files, frozenset()) cmd_1.run(temp_folder) cmd_2.run(temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "piped.txt")) assert_equal(result, "#@!$^")
def test_pformat__simple__running__set_cwd(temp_folder): cmd = AtomicCmd(("sleep", "10"), set_cwd = True) cmd.run(temp_folder) assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n" " Status = Running ...\n" " STDOUT* = 'pipe_sleep_{id}.stdout'\n" " STDERR* = 'pipe_sleep_{id}.stderr'\n" " CWD = '{temp_dir}'>").format(id = id(cmd), temp_dir = temp_folder)) cmd.terminate() cmd.join()
def test_atomiccmd__commit_temp_only(temp_folder): cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT = "bar.txt") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "bar.txt")) cmd.commit(temp_folder) assert_equal(os.listdir(temp_folder), [])
def _do_test_atomiccmd__terminate(temp_folder, raise_on_terminate): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) killpg_was_called = [] def _wrap_killpg(pid, sig): assert_equal(pid, cmd._proc.pid) assert_equal(sig, signal.SIGTERM) killpg_was_called.append(True) if raise_on_terminate: raise OSError("KABOOM!") with Monkeypatch("os.killpg", _wrap_killpg): cmd.terminate() cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"]) assert killpg_was_called
def test_atomiccmd__commit_temp_out(temp_folder): dest, temp = _setup_for_commit(temp_folder, create_cmd = False) cmd = AtomicCmd(("echo", "foo"), OUT_STDOUT = os.path.join(dest, "foo.txt"), TEMP_OUT_FOO = "bar.txt") cmd.run(temp) assert_equal(cmd.join(), [0]) set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3") cmd.commit(temp) assert_equal(os.listdir(temp), []) assert_equal(os.listdir(dest), ["foo.txt"])
def test_pformat__simple__killed(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"]) assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n" " Status = Terminated with signal SIGTERM\n" " STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n" " STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n" " CWD = '{cwd}'>").format(id = id(cmd), temp_dir = temp_folder, cwd = os.getcwd()))
def test_atomiccmd__terminate_sigkill(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd._proc.kill() assert_equal(cmd.join(), ["SIGKILL"])
def test_atomiccmd__commit_while_running(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) assert_raises(CmdError, cmd.commit, temp_folder) cmd.terminate() cmd.join()
def commit(self, temp): AtomicCmd.commit(self, temp) os.remove(os.path.join(destination, "foo.txt"))
def test_atomiccmd__terminate_sigterm(temp_folder): cmd = AtomicCmd(("sleep", "10")) cmd.run(temp_folder) cmd.terminate() assert_equal(cmd.join(), ["SIGTERM"])