def _do_test(cls): cmd_mock_1 = AtomicCmd(("true", ), CHECK_A=id, EXEC_1="false", IN_1="/foo/bar/in_1.file", IN_2="/foo/bar/in_2.file", OUT_1="/bar/foo/out", TEMP_OUT_1="out.log", AUX_A="/aux/fA", AUX_B="/aux/fB") cmd_mock_2 = AtomicCmd(("false", ), CHECK_A=list, EXEC_1="echo", EXEC_2="java", IN_1="/foo/bar/in.file", OUT_1="out.txt") obj = cls([cmd_mock_1, cmd_mock_2]) assert_equal(obj.executables, cmd_mock_1.executables | cmd_mock_2.executables) assert_equal(obj.requirements, cmd_mock_1.requirements | cmd_mock_2.requirements) assert_equal(obj.input_files, cmd_mock_1.input_files | cmd_mock_2.input_files) assert_equal(obj.output_files, cmd_mock_1.output_files | cmd_mock_2.output_files) assert_equal(obj.auxiliary_files, cmd_mock_1.auxiliary_files | cmd_mock_2.auxiliary_files) assert_equal(obj.expected_temp_files, frozenset(["out", "out.txt"])) assert_equal( obj.optional_temp_files, cmd_mock_1.optional_temp_files | cmd_mock_2.optional_temp_files)
def __init__(self, config, reference, input_files, output_file, dependencies): cat_cmds, cat_obj = concatenate_input_bams(config, input_files) cmd_map = AtomicCmd([ "mapDamage", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d", "%(TEMP_DIR)s", "-r", reference ], IN_STDIN=cat_obj, CHECK_VERSION=MAPDAMAGE_VERSION) train_cmds = ParallelCmds(cat_cmds + [cmd_map]) cat_cmds, cat_obj = concatenate_input_bams(config, input_files) cmd_scale = AtomicCmd([ "mapDamage", "--rescale-only", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d", "%(TEMP_DIR)s", "-r", reference, "--rescale-out", "%(OUT_BAM)s" ], IN_STDIN=cat_obj, OUT_BAM=output_file, CHECK_VERSION=MAPDAMAGE_VERSION) rescale_cmds = ParallelCmds(cat_cmds + [cmd_scale]) description = "<mapDamageRescale: %i file(s) -> '%s'>" % ( len(input_files), output_file) CommandNode.__init__(self, command=SequentialCmds([train_cmds, rescale_cmds]), description=description, dependencies=dependencies)
def test_sequential_commands__abort_on_error_1(temp_folder): cmd_1 = AtomicCmd("false") cmd_2 = AtomicCmd(("sleep", 10)) cmd_3 = AtomicCmd(("sleep", 10)) cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(temp_folder) assert_equal(cmds.join(), [1, None, None])
def test_sequential_commands__abort_on_error_3(temp_folder): cmd_1 = AtomicCmd("true") cmd_2 = AtomicCmd("true") cmd_3 = AtomicCmd("false") cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(temp_folder) assert_equal(cmds.join(), [0, 0, 1])
def _do_test_parallel_commands__ready_two(first, second, result): cmd_mock_1 = flexmock(AtomicCmd(["ls"])) cmd_mock_1.should_receive('ready').and_return(first).at_least.once cmd_mock_2 = flexmock(AtomicCmd(["ls"])) cmd_mock_2.should_receive('ready').and_return(second) cmds = ParallelCmds([cmd_mock_1, cmd_mock_2]) assert_equal(cmds.ready(), result)
def test_pformat__atomiccmd__simple_with_stdin__cmd(): cmd_1 = AtomicCmd("gzip", OUT_STDOUT=AtomicCmd.PIPE) cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1) assert_equal(pformat(cmd_2), ("<Command = ['gzip']\n" " STDIN = <PIPE>\n" " STDOUT* = '${TEMP_DIR}/pipe_gzip_%i.stdout'\n" " STDERR* = '${TEMP_DIR}/pipe_gzip_%i.stderr'>") % (id(cmd_2), id(cmd_2)))
def _setup_mocks_for_failure(*do_mocks): results = [] for do_mock in do_mocks: if do_mock: mock = flexmock(AtomicCmd(("sleep", 10))) mock.should_receive('terminate') mock.should_receive('join').and_return(['SIGTERM']) else: mock = AtomicCmd("false") results.append(mock) return results
def test_atomiccmd__piping_temp(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE) assert_equal(cmd_1.output_files, frozenset()) cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt") assert_equal(cmd_2.input_files, frozenset()) cmd_1.run(temp_folder) cmd_2.run(temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2.join(), [0]) result = get_file_contents(os.path.join(temp_folder, "piped.txt")) assert_equal(result, "#@!$^")
def test_atomiccmd__piping_is_only_allowed_once(temp_folder): cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"], OUT_STDOUT=AtomicCmd.PIPE) cmd_2a = AtomicCmd(["grep", "foo"], IN_STDIN=cmd_1) cmd_2b = AtomicCmd(["grep", "bar"], IN_STDIN=cmd_1) cmd_1.run(temp_folder) cmd_2a.run(temp_folder) assert_raises(CmdError, cmd_2b.run, temp_folder) assert_equal(cmd_1.join(), [0]) assert_equal(cmd_2a.join(), [0]) assert_equal(cmd_2b.join(), [None])
def _do_test_pformat__sets__simple(cls, description): cmd_1 = AtomicCmd(("echo", "foo"), OUT_STDOUT=AtomicCmd.PIPE) cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1) cmd = cls((cmd_1, cmd_2)) assert_equal(pformat(cmd), ("<{description}:\n" " - <00> Command = ['echo', 'foo']\n" " STDOUT = <01>\n" " STDERR* = '${{TEMP_DIR}}/pipe_echo_{cmd_1_id}.stderr'\n" " - <01> Command = ['gzip']\n" " STDIN = <00>\n" " STDOUT* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2_id}.stdout'\n" " STDERR* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2_id}.stderr'>") \ .format(description = description, cmd_1_id = id(cmd_1), cmd_2_id = id(cmd_2)))
def test_atomiccmd__paths__key(temp_folder): cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"), OUT_STDOUT=AtomicCmd.PIPE) cmd.run(temp_folder) path = cmd._proc.stdout.read() assert os.path.samefile(temp_folder, path), (temp_folder, path) assert_equal(cmd.join(), [0])
def test_atomiccmd__paths_non_str(temp_folder): cmd = AtomicCmd(("touch", 1234), OUT_FOO = "1234", set_cwd = True) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "1234"))
def __init__(self, config, reference, input_files, output_directory, dependencies): cat_cmds, cat_obj = concatenate_input_bams(config, input_files) cmd_map = AtomicCmd( [ "mapDamage", "--no-stats", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d", "%(TEMP_DIR)s", "-r", reference ], IN_STDIN=cat_obj, OUT_FREQ_3p=os.path.join(output_directory, "3pGtoA_freq.txt"), OUT_FREQ_5p=os.path.join(output_directory, "5pCtoT_freq.txt"), OUT_COMP_USER=os.path.join(output_directory, "dnacomp.txt"), OUT_PLOT_FRAG=os.path.join(output_directory, "Fragmisincorporation_plot.pdf"), OUT_PLOT_LEN=os.path.join(output_directory, "Length_plot.pdf"), OUT_LENGTH=os.path.join(output_directory, "lgdistribution.txt"), OUT_MISINCORP=os.path.join(output_directory, "misincorporation.txt"), OUT_LOG=os.path.join(output_directory, "Runtime_log.txt"), CHECK_VERSION=MAPDAMAGE_VERSION) description = "<mapDamage: %i file(s) -> '%s'>" % (len(input_files), output_directory) CommandNode.__init__(self, command=ParallelCmds(cat_cmds + [cmd_map]), description=description, dependencies=dependencies)
def test_pformat__atomiccmd__simple_with_temp_stdout(): cmd = AtomicCmd(("echo", "Oil. Oil."), TEMP_OUT_STDOUT="dm") assert_equal(pformat(cmd), ("<Command = ['echo', 'Oil. Oil.']\n" " STDOUT* = '${TEMP_DIR}/dm'\n" " STDERR* = '${TEMP_DIR}/pipe_echo_%i.stderr'>") % (id(cmd), ))
def test_atomiccmd__ready(temp_folder): cmd = AtomicCmd("ls") assert_equal(cmd.join(), [None]) assert not cmd.ready() cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert cmd.ready()
def __init__(self, infile, outfile, genome, from_start=0, from_end=0, strand_relative=False, dependencies=()): if type(from_start) != type(from_end): raise ValueError( "'from_start' and 'from_end' should be of same type!") call = [ "slopBed", "-i", "%(IN_FILE)s", "-g", "%(IN_GENOME)s", "-l", str(from_start), "-r", str(from_end) ] if strand_relative: call.append("-s") if type(from_start) is float: call.append("-pct") command = AtomicCmd(call, IN_FILE=infile, IN_GENOME=genome, OUT_STDOUT=outfile) description = "<SlopBed: '%s' -> '%s'>" % (infile, outfile) CommandNode.__init__(self, description=description, command=command, dependencies=dependencies)
def __init__(self, config, d_make, bedn, mappa, unique, dependencies=()): inbedfile = d_make.bedfiles[bedn] basename, extension = os.path.splitext(os.path.basename(inbedfile)) bname = "{}_MappaOnly{}".format(basename, extension) dest = os.path.join(config.temp_local, bname) d_make.bedfiles[bedn] = dest call1 = [ "python", os.path.join(PREFIX, "intersectmappabed.py"), "%(IN_BED)s", "%(IN_MAPPA)s", str(unique), "%(OUT_DEST)s" ] cmd = AtomicCmd(call1, IN_BED=inbedfile, IN_MAPPA=mappa, OUT_DEST=dest, CHECK_VERSION=PYTHON_VERSION) description = ("<CLEANBEDFILES: '%s' -> '%s', Uniqueness: '%s'>" % (inbedfile, dest, unique)) CommandNode.__init__(self, description=description, command=cmd, dependencies=dependencies)
def test_pformat__simple(): cmd = AtomicCmd(("touch", "something")) assert_equal(pformat(cmd), ("<Command = ['touch', 'something']\n" " STDOUT* = '${TEMP_DIR}/pipe_touch_%i.stdout'\n" " STDERR* = '${TEMP_DIR}/pipe_touch_%i.stderr'>") % (id(cmd), id(cmd)))
def test_pformat__atomiccmd__simple_with_stdout_pipe(): cmd = AtomicCmd(("echo", "!"), OUT_STDOUT=AtomicCmd.PIPE) assert_equal(pformat(cmd), ("<Command = ['echo', '!']\n" " STDOUT = <PIPE>\n" " STDERR* = '${TEMP_DIR}/pipe_echo_%i.stderr'>") % (id(cmd), ))
def test_atomicmcd__exec__reqobj(): reqobj = RequirementObj(call=("echo", "version"), search="version", pprint="{0}", checks=str) cmd = AtomicCmd("true", CHECK_VERSION=reqobj) assert_equal(cmd.requirements, frozenset([reqobj]))
def test_atomiccmd__commit_temp_only(temp_folder): cmd = AtomicCmd(("echo", "foo"), TEMP_OUT_STDOUT="bar.txt") cmd.run(temp_folder) assert_equal(cmd.join(), [0]) assert os.path.exists(os.path.join(temp_folder, "bar.txt")) cmd.commit(temp_folder) assert_equal(os.listdir(temp_folder), [])
def __init__(self, config, reference, intervals, infiles, outfile, dependencies=()): self._basename = os.path.basename(outfile) infiles = safe_coerce_to_tuple(infiles) jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar") command = AtomicJavaCmdBuilder(jar_file, jre_options=config.jre_options) command.set_option("-T", "IndelRealigner") command.set_option("-R", "%(IN_REFERENCE)s") command.set_option("-targetIntervals", "%(IN_INTERVALS)s") command.set_option("-o", "%(OUT_BAMFILE)s") command.set_option("--bam_compression", 0) command.set_option("--disable_bam_indexing") _set_input_files(command, infiles) command.set_kwargs(IN_REFERENCE=reference, IN_REF_DICT=fileutils.swap_ext(reference, ".dict"), IN_INTERVALS=intervals, OUT_BAMFILE=outfile, CHECK_GATK=_get_gatk_version_check(config)) calmd = AtomicCmd(["samtools", "calmd", "-b", "%(TEMP_IN_BAM)s", "%(IN_REF)s"], TEMP_IN_BAM=self._basename, IN_REF=reference, TEMP_OUT_STDOUT=self._basename + ".calmd", CHECK_VERSION=SAMTOOLS_VERSION) description = "<Indel Realigner (aligning): %s -> %r>" \ % (describe_files(infiles), outfile) CommandNode.__init__(self, description=description, command=ParallelCmds([command.finalize(), calmd]), dependencies=dependencies)
def test_atomiccmd__terminate_race_condition(temp_folder): cmd = AtomicCmd("true") cmd.run(temp_folder) while cmd._proc.poll() is None: pass cmd.terminate() assert_equal(cmd.join(), [0])
def test_pformat__atomiccmd__simple_with_stdout(): cmd = AtomicCmd(("echo", "Water. Water."), OUT_STDOUT="/dev/ls") assert_equal(pformat(cmd), ("<Command = ['echo', 'Water. Water.']\n" " STDOUT = '${TEMP_DIR}/ls'\n" " STDERR* = '${TEMP_DIR}/pipe_echo_%i.stderr'>") % (id(cmd), ))
def test_atomiccmd__commit_before_join(temp_folder): cmd = AtomicCmd(("sleep", "0.1")) cmd.run(temp_folder) while cmd._proc.poll() is None: pass assert_raises(CmdError, cmd.commit, temp_folder) cmd.join()
def test_pformat__atomiccmd__simple_with_temp_infile(): cmd = AtomicCmd(("cat", "%(TEMP_IN_FILE)s"), TEMP_IN_FILE="infile.txt") assert_equal(pformat(cmd), ("<Command = ['cat', '${TEMP_DIR}/infile.txt']\n" " STDOUT* = '${TEMP_DIR}/pipe_cat_%i.stdout'\n" " STDERR* = '${TEMP_DIR}/pipe_cat_%i.stderr'>") % (id(cmd), id(cmd)))
def finalize(self): """Creates an AtomicCmd object based on the AtomicParam object. Once finalized, the AtomicCmdBuilder cannot be modified further.""" if not self._object: self._object = AtomicCmd(self.call, **self.kwargs) return self._object
def test_pformat__atomiccmd__simple_with_infile(): cmd = AtomicCmd(("cat", "%(IN_SOMETHING)s"), IN_SOMETHING="/etc/fstab") assert_equal(pformat(cmd), ("<Command = ['cat', '/etc/fstab']\n" " STDOUT* = '${TEMP_DIR}/pipe_cat_%i.stdout'\n" " STDERR* = '${TEMP_DIR}/pipe_cat_%i.stderr'>") % (id(cmd), id(cmd)))
def test_pformat__atomiccmd__simple_with_temp_outfile(): cmd = AtomicCmd(("touch", "%(TEMP_OUT_RC)s"), TEMP_OUT_RC="bashrc") assert_equal(pformat(cmd), ("<Command = ['touch', '${TEMP_DIR}/bashrc']\n" " STDOUT* = '${TEMP_DIR}/pipe_touch_%i.stdout'\n" " STDERR* = '${TEMP_DIR}/pipe_touch_%i.stderr'>") % (id(cmd), id(cmd)))
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs): cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs) cmd.run(temp_folder) assert_equal(cmd.join(), [0]) result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd)))) result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd)))) assert_equal(result_out, "STDOUT!") assert_equal(result_err, "STDERR!")