def test_sequential_commands__abort_on_error_1(temp_folder): cmd_1 = AtomicCmd("false") cmd_2 = AtomicCmd(("sleep", 10)) cmd_3 = AtomicCmd(("sleep", 10)) cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(temp_folder) assert_equal(cmds.join(), [1, None, None])
def test_sequential_commands__abort_on_error_3(tmp_path): cmd_1 = AtomicCmd("true") cmd_2 = AtomicCmd("true") cmd_3 = AtomicCmd("false") cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(tmp_path) assert cmds.join() == [0, 0, 1]
def test_sequential_commands__abort_on_error_1(tmp_path): cmd_1 = AtomicCmd("false") cmd_2 = AtomicCmd(("sleep", 10)) cmd_3 = AtomicCmd(("sleep", 10)) cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(tmp_path) assert cmds.join() == [1, None, None]
def test_sequential_commands__abort_on_error_3(temp_folder): cmd_1 = AtomicCmd("true") cmd_2 = AtomicCmd("true") cmd_3 = AtomicCmd("false") cmds = SequentialCmds([cmd_1, cmd_2, cmd_3]) cmds.run(temp_folder) assert_equal(cmds.join(), [0, 0, 1])
def __init__(self, samples, prefix, output_prefix, dependencies=()): abs_prefix = os.path.abspath(prefix) basename = os.path.basename(output_prefix) # TreeMix plots with migration edges cmd_1 = self._plot_command(prefix, "plot_tree", abs_prefix, "%(IN_SAMPLES)s", "%(TEMP_OUT_PREFIX)s", IN_SAMPLES=samples, TEMP_OUT_PREFIX=basename + "_tree", OUT_PDF=output_prefix + "_tree.pdf", OUT_PNG=output_prefix + "_tree.png") # Heatmap showing TreeMix residuals cmd_2 = self._plot_command(prefix, "plot_residuals", abs_prefix, "%(IN_SAMPLES)s", "%(TEMP_OUT_PREFIX)s", IN_SAMPLES=samples, TEMP_OUT_PREFIX=basename + "_residuals", OUT_PDF=output_prefix + "_residuals.pdf", OUT_PNG=output_prefix + "_residuals.png") # Text file containing % of variance explained by model cmd_3 = self._plot_command(prefix, "variance", abs_prefix, "%(OUT_TXT)s", OUT_TXT=output_prefix + "_variance.txt") CommandNode.__init__(self, description="<PlotTreemix -> '%s.*'>" % (output_prefix,), command=SequentialCmds((cmd_1, cmd_2, cmd_3)), dependencies=dependencies)
def __init__(self, infile, index_format='.bai', dependencies=()): basename = os.path.basename(infile) if index_format == '.bai': samtools_version = SAMTOOLS_VERSION samtools_call = ["samtools", "index", "%(TEMP_IN_BAM)s"] elif index_format == '.csi': samtools_version = SAMTOOLS_VERSION_1x samtools_call = ["samtools", "index", "-c", "%(TEMP_IN_BAM)s"] else: raise ValueError("Unknown format type %r; expected .bai or .csi" % (index_format, )) cmd_link = AtomicCmd(["ln", "-s", "%(IN_BAM)s", "%(TEMP_OUT_BAM)s"], IN_BAM=infile, TEMP_OUT_BAM=basename, set_cwd=True) cmd_index = AtomicCmd(samtools_call, TEMP_IN_BAM=basename, CHECK_SAM=samtools_version) cmd_rename = AtomicCmd(["mv", "%(TEMP_IN_BAM)s", "%(OUT_BAM)s"], TEMP_IN_BAM=basename + index_format, OUT_BAM=swap_ext(infile, index_format)) commands = SequentialCmds((cmd_link, cmd_index, cmd_rename)) CommandNode.__init__(self, description="<BAMIndex (%s): '%s'>" % (index_format[1:].upper(), infile), command=commands, dependencies=dependencies)
def test_pformat__sets__nested(): cmd_1 = AtomicCmd(("echo", "foo"), OUT_STDOUT=AtomicCmd.PIPE) cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1) cmd_3 = AtomicCmd("sha1sum") set_1 = ParallelCmds((cmd_1, cmd_2)) set_2 = SequentialCmds((set_1, cmd_3)) assert pformat(set_2) == ( "Sequential processes:\n" " Parallel processes:\n" " Process 1:\n" " Command = echo foo\n" " STDOUT = Piped to process 2\n" " STDERR* = '${{TEMP_DIR}}/pipe_echo_{cmd_1}.stderr'\n" "\n" " Process 2:\n" " Command = gzip\n" " STDIN = Piped from process 1\n" " STDOUT* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2}.stdout'\n" " STDERR* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2}.stderr'\n" "\n" " Process 3:\n" " Command = sha1sum\n" " STDOUT* = '${{TEMP_DIR}}/pipe_sha1sum_{cmd_3}.stdout'\n" " STDERR* = '${{TEMP_DIR}}/pipe_sha1sum_{cmd_3}.stderr'").format( cmd_1=id(cmd_1), cmd_2=id(cmd_2), cmd_3=id(cmd_3))
def test_sequential_commands__atomiccmds(): mock = Mock() cmd_1 = AtomicCmd(["ls"]) cmd_1.run = mock.run_1 cmd_1.join = mock.join_1 cmd_1.join.return_value = [0] cmd_2 = AtomicCmd(["ls"]) cmd_2.run = mock.run_2 cmd_2.join = mock.join_2 cmd_2.join.return_value = [0] cmd_3 = AtomicCmd(["ls"]) cmd_3.run = mock.run_3 cmd_3.join = mock.join_3 cmd_3.join.return_value = [0] cmds = SequentialCmds((cmd_1, cmd_2, cmd_3)) assert not cmds.ready() cmds.run("xTMPx") assert cmds.ready() assert cmds.join() == [0, 0, 0] assert mock.mock_calls == [ call.run_1("xTMPx"), call.join_1(), call.run_2("xTMPx"), call.join_2(), call.run_3("xTMPx"), call.join_3(), call.join_1(), call.join_2(), call.join_3(), ]
def __init__(self, output_prefix, tfam, tped, indep_filter=None, indep_parameters=None, plink_parameters=None, dependencies=()): assert indep_filter in ('indep', 'indep-pairphase', 'indep-pairwise'), indep_filter assert len(indep_parameters) == 3, indep_parameters parameters = self._parse_parameters(plink_parameters) plink_cmd = [ "plink", "--noweb", "--tped", "%(IN_TPED)s", "--tfam", "%(IN_TFAM)s", "--out", "%(TEMP_OUT_PREFIX)s", '--' + indep_filter ] plink_cmd.extend(indep_parameters) plink_cmd.extend(parameters) cmd_indep = AtomicCmd(plink_cmd, IN_TFAM=tfam, IN_TPED=tped, TEMP_OUT_PREFIX="indep", TEMP_OUT_LOG="indep.log", TEMP_OUT_NOSEX="indep.nosex", TEMP_OUT_PRUNE_IN="indep.prune.in", TEMP_OUT_PRUNE_OUT="indep.prune.out", set_cwd=True) basename = os.path.basename(output_prefix) cmd_filter = AtomicCmd([ "plink", "--noweb", "--make-bed", "--tped", "%(IN_TPED)s", "--tfam", "%(IN_TFAM)s", "--extract", "%(TEMP_IN_PRUNE)s", "--out", "%(TEMP_OUT_PREFIX)s" ] + parameters, IN_TFAM=tfam, IN_TPED=tped, TEMP_OUT_PREFIX=basename, TEMP_IN_PRUNE="indep.prune.in", TEMP_OUT_NOSEX=basename + ".nosex", TEMP_OUT_LOG=basename + ".log", OUT_LOG=output_prefix + ".log", OUT_BED=output_prefix + ".bed", OUT_BIM=output_prefix + ".bim", OUT_FAM=output_prefix + ".fam", set_cwd=True) CommandNode.__init__(self, description="<BuildFilteredBEDFiles -> '%s.*'>" % (output_prefix, ), command=SequentialCmds((cmd_indep, cmd_filter)), dependencies=dependencies)
def __init__(self, input_prefix, output_prefix, tfam, parameters=None, dependencies=()): basename = os.path.basename(output_prefix) plink_cmd = [ "plink", "--freq", "--missing", "--noweb", "--bfile", os.path.abspath(input_prefix), "--within", "%(TEMP_OUT_CLUST)s", "--out", "%(TEMP_OUT_PREFIX)s", ] if parameters: plink_cmd.extend(parameters.split()) plink = AtomicCmd( plink_cmd, IN_BED=input_prefix + ".bed", IN_BIM=input_prefix + ".bim", IN_FAM=input_prefix + ".fam", TEMP_OUT_CLUST="samples.clust", TEMP_OUT_IMISS=basename + ".imiss", TEMP_OUT_LMISS=basename + ".lmiss", OUT_NOSEX=output_prefix + ".frq.strat.nosex", OUT_LOG=output_prefix + ".frq.strat.log", TEMP_OUT_PREFIX=basename, CHECK_VERSION=PLINK_VERSION, set_cwd=True, ) gzip = AtomicCmd( ["gzip", "%(TEMP_IN_FREQ)s"], TEMP_IN_FREQ=basename + ".frq.strat", OUT_FREQ=output_prefix + ".frq.strat.gz", ) self._tfam = tfam self._basename = basename CommandNode.__init__( self, description="<BuildFreqFiles -> '%s.*'" % (output_prefix, ), command=SequentialCmds((plink, gzip)), dependencies=dependencies, )
def test_sequential_commands__atomiccmds(): mocks = [] for _ in range(3): cmd_mock = flexmock(AtomicCmd(["ls"])) cmd_mock.should_receive('run').with_args("xTMPx").once cmd_mock.should_receive('join').with_args().and_return([0]).twice mocks.append(cmd_mock) cmds = SequentialCmds(mocks) assert not cmds.ready() cmds.run("xTMPx") assert cmds.ready() assert_equal(cmds.join(), [0, 0, 0])
def __init__(self, control_file, sequence_file, trees_file, output_tar, exclude_groups=(), dependencies=()): self._exclude_groups = safe_coerce_to_frozenset(exclude_groups) self._control_file = control_file self._sequence_file = sequence_file self._trees_file = trees_file paml_cmd = AtomicCmd( ["codeml", "template.ctl"], IN_CONTROL_FILE=control_file, IN_SEQUENCE_FILE=sequence_file, IN_TREES_FILE=trees_file, TEMP_OUT_CTL="template.ctl", TEMP_OUT_SEQS="template.seqs", TEMP_OUT_TREES="template.trees", TEMP_OUT_STDOUT="template.stdout", TEMP_OUT_STDERR="template.stderr", TEMP_OUT_4FOLD="4fold.nuc", IN_STDIN="/dev/null", # Prevent promts from blocking set_cwd=True, **CodemlNode._get_codeml_files("TEMP_OUT_CODEML")) tar_pairs = CodemlNode._get_codeml_files("TEMP_IN_CODEML") tar_files = ["%%(%s)s" % (key, ) for key in tar_pairs] tar_cmd = AtomicCmd(["tar", "cvzf", "%(OUT_FILE)s"] + tar_files, OUT_FILE=output_tar, set_cwd=True, **tar_pairs) CommandNode.__init__(self, description="<CodemlNode: %r -> %r>" % (sequence_file, output_tar), command=SequentialCmds([paml_cmd, tar_cmd]), dependencies=dependencies)
def test_parallel_commands__reject_sequential(): command = AtomicCmd(["ls"]) seqcmd = SequentialCmds([command]) assert_raises(CmdError, ParallelCmds, [seqcmd])
def test_sequential_commands__reject_noncommand(): SequentialCmds([object()])
def test_sequential_commands__reject_empty_commandset(): with pytest.raises(CmdError): SequentialCmds([])
def test_sequential_commands__reject_noncommand(): with pytest.raises(CmdError): SequentialCmds([object()])
def test_sequential_commands__accept_sequential(): command = AtomicCmd(["ls"]) seqcmd = SequentialCmds([command]) SequentialCmds([seqcmd])
def test_sequential_commands__accept_parallel(): command = AtomicCmd(["ls"]) parcmd = ParallelCmds([command]) SequentialCmds([parcmd])
def test_sequential_commands__reject_empty_commandset(): SequentialCmds([])
def test_parallel_commands__reject_sequential(): command = AtomicCmd(["ls"]) seqcmd = SequentialCmds([command]) with pytest.raises(CmdError): ParallelCmds([seqcmd])