def test_sequential_commands__abort_on_error_1(temp_folder):
cmd_1 = AtomicCmd("false")
cmd_2 = AtomicCmd(("sleep", 10))
cmd_3 = AtomicCmd(("sleep", 10))
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(temp_folder)
assert_equal(cmds.join(), [1, None, None])
def test_sequential_commands__abort_on_error_3(tmp_path):
cmd_1 = AtomicCmd("true")
cmd_2 = AtomicCmd("true")
cmd_3 = AtomicCmd("false")
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(tmp_path)
assert cmds.join() == [0, 0, 1]
def test_sequential_commands__abort_on_error_1(tmp_path):
cmd_1 = AtomicCmd("false")
cmd_2 = AtomicCmd(("sleep", 10))
cmd_3 = AtomicCmd(("sleep", 10))
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(tmp_path)
assert cmds.join() == [1, None, None]
def test_sequential_commands__abort_on_error_3(temp_folder):
cmd_1 = AtomicCmd("true")
cmd_2 = AtomicCmd("true")
cmd_3 = AtomicCmd("false")
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(temp_folder)
assert_equal(cmds.join(), [0, 0, 1])
def test_sequential_commands__abort_on_error_1(temp_folder):
cmd_1 = AtomicCmd("false")
cmd_2 = AtomicCmd(("sleep", 10))
cmd_3 = AtomicCmd(("sleep", 10))
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(temp_folder)
assert_equal(cmds.join(), [1, None, None])
def test_sequential_commands__abort_on_error_3(temp_folder):
cmd_1 = AtomicCmd("true")
cmd_2 = AtomicCmd("true")
cmd_3 = AtomicCmd("false")
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(temp_folder)
assert_equal(cmds.join(), [0, 0, 1])
def __init__(self, samples, prefix, output_prefix, dependencies=()):
abs_prefix = os.path.abspath(prefix)
basename = os.path.basename(output_prefix)
# TreeMix plots with migration edges
cmd_1 = self._plot_command(prefix, "plot_tree", abs_prefix,
"%(IN_SAMPLES)s", "%(TEMP_OUT_PREFIX)s",
IN_SAMPLES=samples,
TEMP_OUT_PREFIX=basename + "_tree",
OUT_PDF=output_prefix + "_tree.pdf",
OUT_PNG=output_prefix + "_tree.png")
# Heatmap showing TreeMix residuals
cmd_2 = self._plot_command(prefix, "plot_residuals", abs_prefix,
"%(IN_SAMPLES)s", "%(TEMP_OUT_PREFIX)s",
IN_SAMPLES=samples,
TEMP_OUT_PREFIX=basename + "_residuals",
OUT_PDF=output_prefix + "_residuals.pdf",
OUT_PNG=output_prefix + "_residuals.png")
# Text file containing % of variance explained by model
cmd_3 = self._plot_command(prefix, "variance", abs_prefix,
"%(OUT_TXT)s",
OUT_TXT=output_prefix + "_variance.txt")
CommandNode.__init__(self,
description="<PlotTreemix -> '%s.*'>"
% (output_prefix,),
command=SequentialCmds((cmd_1, cmd_2, cmd_3)),
dependencies=dependencies)
def __init__(self, infile, index_format='.bai', dependencies=()):
basename = os.path.basename(infile)
if index_format == '.bai':
samtools_version = SAMTOOLS_VERSION
samtools_call = ["samtools", "index", "%(TEMP_IN_BAM)s"]
elif index_format == '.csi':
samtools_version = SAMTOOLS_VERSION_1x
samtools_call = ["samtools", "index", "-c", "%(TEMP_IN_BAM)s"]
else:
raise ValueError("Unknown format type %r; expected .bai or .csi" %
(index_format, ))
cmd_link = AtomicCmd(["ln", "-s", "%(IN_BAM)s", "%(TEMP_OUT_BAM)s"],
IN_BAM=infile,
TEMP_OUT_BAM=basename,
set_cwd=True)
cmd_index = AtomicCmd(samtools_call,
TEMP_IN_BAM=basename,
CHECK_SAM=samtools_version)
cmd_rename = AtomicCmd(["mv", "%(TEMP_IN_BAM)s", "%(OUT_BAM)s"],
TEMP_IN_BAM=basename + index_format,
OUT_BAM=swap_ext(infile, index_format))
commands = SequentialCmds((cmd_link, cmd_index, cmd_rename))
CommandNode.__init__(self,
description="<BAMIndex (%s): '%s'>" %
(index_format[1:].upper(), infile),
command=commands,
dependencies=dependencies)
def test_pformat__sets__nested():
cmd_1 = AtomicCmd(("echo", "foo"), OUT_STDOUT=AtomicCmd.PIPE)
cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1)
cmd_3 = AtomicCmd("sha1sum")
set_1 = ParallelCmds((cmd_1, cmd_2))
set_2 = SequentialCmds((set_1, cmd_3))
assert pformat(set_2) == (
"Sequential processes:\n"
" Parallel processes:\n"
" Process 1:\n"
" Command = echo foo\n"
" STDOUT = Piped to process 2\n"
" STDERR* = '${{TEMP_DIR}}/pipe_echo_{cmd_1}.stderr'\n"
"\n"
" Process 2:\n"
" Command = gzip\n"
" STDIN = Piped from process 1\n"
" STDOUT* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2}.stdout'\n"
" STDERR* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2}.stderr'\n"
"\n"
" Process 3:\n"
" Command = sha1sum\n"
" STDOUT* = '${{TEMP_DIR}}/pipe_sha1sum_{cmd_3}.stdout'\n"
" STDERR* = '${{TEMP_DIR}}/pipe_sha1sum_{cmd_3}.stderr'").format(
cmd_1=id(cmd_1), cmd_2=id(cmd_2), cmd_3=id(cmd_3))
def test_sequential_commands__atomiccmds():
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.run = mock.run_1
cmd_1.join = mock.join_1
cmd_1.join.return_value = [0]
cmd_2 = AtomicCmd(["ls"])
cmd_2.run = mock.run_2
cmd_2.join = mock.join_2
cmd_2.join.return_value = [0]
cmd_3 = AtomicCmd(["ls"])
cmd_3.run = mock.run_3
cmd_3.join = mock.join_3
cmd_3.join.return_value = [0]
cmds = SequentialCmds((cmd_1, cmd_2, cmd_3))
assert not cmds.ready()
cmds.run("xTMPx")
assert cmds.ready()
assert cmds.join() == [0, 0, 0]
assert mock.mock_calls == [
call.run_1("xTMPx"),
call.join_1(),
call.run_2("xTMPx"),
call.join_2(),
call.run_3("xTMPx"),
call.join_3(),
call.join_1(),
call.join_2(),
call.join_3(),
]
def __init__(self,
output_prefix,
tfam,
tped,
indep_filter=None,
indep_parameters=None,
plink_parameters=None,
dependencies=()):
assert indep_filter in ('indep', 'indep-pairphase',
'indep-pairwise'), indep_filter
assert len(indep_parameters) == 3, indep_parameters
parameters = self._parse_parameters(plink_parameters)
plink_cmd = [
"plink", "--noweb", "--tped", "%(IN_TPED)s", "--tfam",
"%(IN_TFAM)s", "--out", "%(TEMP_OUT_PREFIX)s", '--' + indep_filter
]
plink_cmd.extend(indep_parameters)
plink_cmd.extend(parameters)
cmd_indep = AtomicCmd(plink_cmd,
IN_TFAM=tfam,
IN_TPED=tped,
TEMP_OUT_PREFIX="indep",
TEMP_OUT_LOG="indep.log",
TEMP_OUT_NOSEX="indep.nosex",
TEMP_OUT_PRUNE_IN="indep.prune.in",
TEMP_OUT_PRUNE_OUT="indep.prune.out",
set_cwd=True)
basename = os.path.basename(output_prefix)
cmd_filter = AtomicCmd([
"plink", "--noweb", "--make-bed", "--tped", "%(IN_TPED)s",
"--tfam", "%(IN_TFAM)s", "--extract", "%(TEMP_IN_PRUNE)s", "--out",
"%(TEMP_OUT_PREFIX)s"
] + parameters,
IN_TFAM=tfam,
IN_TPED=tped,
TEMP_OUT_PREFIX=basename,
TEMP_IN_PRUNE="indep.prune.in",
TEMP_OUT_NOSEX=basename + ".nosex",
TEMP_OUT_LOG=basename + ".log",
OUT_LOG=output_prefix + ".log",
OUT_BED=output_prefix + ".bed",
OUT_BIM=output_prefix + ".bim",
OUT_FAM=output_prefix + ".fam",
set_cwd=True)
CommandNode.__init__(self,
description="<BuildFilteredBEDFiles -> '%s.*'>" %
(output_prefix, ),
command=SequentialCmds((cmd_indep, cmd_filter)),
dependencies=dependencies)
def __init__(self,
input_prefix,
output_prefix,
tfam,
parameters=None,
dependencies=()):
basename = os.path.basename(output_prefix)
plink_cmd = [
"plink",
"--freq",
"--missing",
"--noweb",
"--bfile",
os.path.abspath(input_prefix),
"--within",
"%(TEMP_OUT_CLUST)s",
"--out",
"%(TEMP_OUT_PREFIX)s",
]
if parameters:
plink_cmd.extend(parameters.split())
plink = AtomicCmd(
plink_cmd,
IN_BED=input_prefix + ".bed",
IN_BIM=input_prefix + ".bim",
IN_FAM=input_prefix + ".fam",
TEMP_OUT_CLUST="samples.clust",
TEMP_OUT_IMISS=basename + ".imiss",
TEMP_OUT_LMISS=basename + ".lmiss",
OUT_NOSEX=output_prefix + ".frq.strat.nosex",
OUT_LOG=output_prefix + ".frq.strat.log",
TEMP_OUT_PREFIX=basename,
CHECK_VERSION=PLINK_VERSION,
set_cwd=True,
)
gzip = AtomicCmd(
["gzip", "%(TEMP_IN_FREQ)s"],
TEMP_IN_FREQ=basename + ".frq.strat",
OUT_FREQ=output_prefix + ".frq.strat.gz",
)
self._tfam = tfam
self._basename = basename
CommandNode.__init__(
self,
description="<BuildFreqFiles -> '%s.*'" % (output_prefix, ),
command=SequentialCmds((plink, gzip)),
dependencies=dependencies,
)
def test_sequential_commands__atomiccmds():
mocks = []
for _ in range(3):
cmd_mock = flexmock(AtomicCmd(["ls"]))
cmd_mock.should_receive('run').with_args("xTMPx").once
cmd_mock.should_receive('join').with_args().and_return([0]).twice
mocks.append(cmd_mock)
cmds = SequentialCmds(mocks)
assert not cmds.ready()
cmds.run("xTMPx")
assert cmds.ready()
assert_equal(cmds.join(), [0, 0, 0])
def test_sequential_commands__atomiccmds():
mocks = []
for _ in range(3):
cmd_mock = flexmock(AtomicCmd(["ls"]))
cmd_mock.should_receive('run').with_args("xTMPx").once
cmd_mock.should_receive('join').with_args().and_return([0]).twice
mocks.append(cmd_mock)
cmds = SequentialCmds(mocks)
assert not cmds.ready()
cmds.run("xTMPx")
assert cmds.ready()
assert_equal(cmds.join(), [0, 0, 0])
def __init__(self,
control_file,
sequence_file,
trees_file,
output_tar,
exclude_groups=(),
dependencies=()):
self._exclude_groups = safe_coerce_to_frozenset(exclude_groups)
self._control_file = control_file
self._sequence_file = sequence_file
self._trees_file = trees_file
paml_cmd = AtomicCmd(
["codeml", "template.ctl"],
IN_CONTROL_FILE=control_file,
IN_SEQUENCE_FILE=sequence_file,
IN_TREES_FILE=trees_file,
TEMP_OUT_CTL="template.ctl",
TEMP_OUT_SEQS="template.seqs",
TEMP_OUT_TREES="template.trees",
TEMP_OUT_STDOUT="template.stdout",
TEMP_OUT_STDERR="template.stderr",
TEMP_OUT_4FOLD="4fold.nuc",
IN_STDIN="/dev/null", # Prevent promts from blocking
set_cwd=True,
**CodemlNode._get_codeml_files("TEMP_OUT_CODEML"))
tar_pairs = CodemlNode._get_codeml_files("TEMP_IN_CODEML")
tar_files = ["%%(%s)s" % (key, ) for key in tar_pairs]
tar_cmd = AtomicCmd(["tar", "cvzf", "%(OUT_FILE)s"] + tar_files,
OUT_FILE=output_tar,
set_cwd=True,
**tar_pairs)
CommandNode.__init__(self,
description="<CodemlNode: %r -> %r>" %
(sequence_file, output_tar),
command=SequentialCmds([paml_cmd, tar_cmd]),
dependencies=dependencies)
def test_parallel_commands__reject_sequential():
command = AtomicCmd(["ls"])
seqcmd = SequentialCmds([command])
assert_raises(CmdError, ParallelCmds, [seqcmd])
def test_sequential_commands__reject_noncommand():
SequentialCmds([object()])
def test_sequential_commands__reject_empty_commandset():
with pytest.raises(CmdError):
SequentialCmds([])
def test_sequential_commands__reject_noncommand():
with pytest.raises(CmdError):
SequentialCmds([object()])
def test_sequential_commands__accept_sequential():
command = AtomicCmd(["ls"])
seqcmd = SequentialCmds([command])
SequentialCmds([seqcmd])
def test_sequential_commands__accept_parallel():
command = AtomicCmd(["ls"])
parcmd = ParallelCmds([command])
SequentialCmds([parcmd])
def test_sequential_commands__reject_empty_commandset():
SequentialCmds([])
def test_parallel_commands__reject_sequential():
command = AtomicCmd(["ls"])
seqcmd = SequentialCmds([command])
with pytest.raises(CmdError):
ParallelCmds([seqcmd])