def test_atomiccmd__paths_non_str(temp_folder):
cmd = AtomicCmd(("touch", 1234),
OUT_FOO="1234",
set_cwd=True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "1234"))
def test_atomiccmd__paths__key(temp_folder):
cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"),
OUT_STDOUT=AtomicCmd.PIPE)
cmd.run(temp_folder)
path = cmd._proc.stdout.read()
assert os.path.samefile(temp_folder, path), (temp_folder, path)
assert_equal(cmd.join(), [0])
def test_atomiccmd__commit_before_join(temp_folder):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.join()
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd))))
result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd))))
assert_equal(result_out, "STDOUT!")
assert_equal(result_err, "STDERR!")
def test_pformat__simple__done__set_cwd(temp_folder):
cmd = AtomicCmd("true", set_cwd=True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd), ("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = 'pipe_true_{id}.stdout'\n"
" STDERR* = 'pipe_true_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
def test_atomiccmd__commit_missing_files(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"),
OUT_BAR=os.path.join(destination, "4567"))
cmd.run(temp_folder)
cmd.join()
before = set(os.listdir(temp_folder))
assert_raises(CmdError, cmd.commit, temp_folder)
assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__pipes_stdin(temp_folder):
fname = test_file("fasta_file.fasta")
cmd = AtomicCmd("cat",
IN_STDIN=fname,
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN="infile.fasta",
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, "a\nbc\nd")
def _test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def test_pformat__simple__done(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd),
("Command = true\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd):
cwd = os.getcwd()
cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(cwd, os.getcwd())
expected = temp_folder if set_cwd else cwd
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
assert_equal(paleomix.atomiccmd.command._PROCS, set())
cmd = AtomicCmd("ls")
cmd.run(temp_folder)
ref = iter(paleomix.atomiccmd.command._PROCS).next()
assert ref
assert_equal(ref(), cmd._proc)
assert_equal(cmd.join(), [0])
cmd = func(cmd, temp_folder)
assert ref not in paleomix.atomiccmd.command._PROCS
def test_commandnode_teardown__missing_optional_files(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
TEMP_OUT_BAR="bar.txt",
OUT_STDOUT=os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
node._teardown(None, temp_folder)
assert_equal(os.listdir(temp_folder), [])
assert_equal(os.listdir(destination), ["foo.txt"])
def test_pformat__simple__temp_root_in_arguments(temp_folder):
cmd = AtomicCmd(("echo", "${TEMP_DIR}"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd),
("Command = echo '{temp_dir}'\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_echo_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_echo_{id}.stderr'\n"
"CWD = '{cwd}'")
.format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def test_atomiccmd__piping_is_only_allowed_once(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"],
OUT_STDOUT=AtomicCmd.PIPE)
cmd_2a = AtomicCmd(["grep", "foo"],
IN_STDIN=cmd_1)
cmd_2b = AtomicCmd(["grep", "bar"],
IN_STDIN=cmd_1)
cmd_1.run(temp_folder)
cmd_2a.run(temp_folder)
assert_raises(CmdError, cmd_2b.run, temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2a.join(), [0])
assert_equal(cmd_2b.join(), [None])
def test_atomiccmd__ready(tmp_path):
cmd = AtomicCmd("ls")
assert cmd.join() == [None]
assert not cmd.ready()
cmd.run(tmp_path)
assert cmd.join() == [0]
assert cmd.ready()
def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def test_commandnode_teardown(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
OUT_STDOUT=os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
assert os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert not os.path.exists(os.path.join(destination, "foo.txt"))
node._teardown(None, temp_folder)
assert not os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert os.path.exists(os.path.join(destination, "foo.txt"))
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
# pylint: disable=protected-access
cmd._proc.wait()
assert_equal(pformat(cmd),
("Command = true\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
assert_equal(cmd.join(), [0])
def test_pformat__simple__killed_by_signal(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
# pylint: disable=protected-access
os.killpg(cmd._proc.pid, signal.SIGTERM)
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Terminated with signal SIGTERM\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def test_pformat__simple__killed_by_signal(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
# pylint: disable=protected-access
os.killpg(cmd._proc.pid, signal.SIGTERM)
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Terminated with signal SIGTERM\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected_files = []
for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
if tmpl is not None:
fname = tmpl.format(id(cmd))
result = get_file_contents(os.path.join(temp_folder, fname))
assert_equal(result, text)
expected_files.append(fname)
assert_equal(set(os.listdir(temp_folder)), set(expected_files))
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
# pylint: disable=protected-access
cmd._proc.wait()
assert_equal(pformat(cmd),
("Command = true\n"
"Status = Exited with return-code 0\n"
"STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
assert_equal(cmd.join(), [0])
def _setup_for_commit(tmp_path, create_cmd=True):
destination = tmp_path / "out"
tmp_path = tmp_path / "tmp"
os.makedirs(destination)
os.makedirs(tmp_path)
if not create_cmd:
return destination, tmp_path
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"))
cmd.run(tmp_path)
assert cmd.join() == [0]
return destination, tmp_path, cmd
def test_commandnode_teardown__missing_optional_files(tmp_path):
destination, tmp_path = _setup_temp_folders(tmp_path)
cmd = AtomicCmd(
("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
TEMP_OUT_BAR="bar.txt",
OUT_STDOUT=os.path.join(destination, "foo.txt"),
)
cmd.run(tmp_path)
assert cmd.join() == [0]
node = CommandNode(cmd)
node._teardown(None, tmp_path)
assert os.listdir(tmp_path) == []
assert os.listdir(destination) == ["foo.txt"]
def _setup_for_commit(temp_folder, create_cmd=True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd
def test_atomiccmd__commit_before_join(tmp_path):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(tmp_path)
while cmd._proc.poll() is None:
pass
with pytest.raises(CmdError):
cmd.commit(tmp_path)
cmd.join()
def _setup_for_commit(temp_folder, create_cmd=True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd
def test_pformat__atomiccmd__simple_with_stdin():
cmd = AtomicCmd("gzip", IN_STDIN="/etc/fstab")
assert pformat(cmd) == ("Command = gzip\n"
"STDIN = '/etc/fstab'\n"
"STDOUT* = '${TEMP_DIR}/pipe_gzip_%i.stdout'\n"
"STDERR* = '${TEMP_DIR}/pipe_gzip_%i.stderr'") % (
id(cmd), id(cmd))
def test_commandnode_teardown__extra_files_in_temp(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
OUT_STDOUT=os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
set_file_contents(os.path.join(temp_folder, "bar.txt"), "1 2 3")
temp_files_before = set(os.listdir(temp_folder))
dest_files_before = set(os.listdir(destination))
assert_raises(CmdNodeError, node._teardown, None, temp_folder)
assert_equal(temp_files_before, set(os.listdir(temp_folder)))
assert_equal(dest_files_before, set(os.listdir(destination)))
def test_pformat__simple():
cmd = AtomicCmd(("touch", "something"))
assert_equal(pformat(cmd),
("Command = touch something\n"
"STDOUT* = '${TEMP_DIR}/pipe_touch_%i.stdout'\n"
"STDERR* = '${TEMP_DIR}/pipe_touch_%i.stderr'") %
(id(cmd), id(cmd)))
def test_commandnode_teardown(tmp_path):
destination, tmp_path = _setup_temp_folders(tmp_path)
cmd = AtomicCmd(
("echo", "-n", "1 2 3"),
IN_DUMMY=_EMPTY_FILE,
OUT_STDOUT=os.path.join(destination, "foo.txt"),
)
cmd.run(tmp_path)
assert cmd.join() == [0]
node = CommandNode(cmd)
assert os.path.exists(os.path.join(tmp_path, "foo.txt"))
assert not os.path.exists(os.path.join(destination, "foo.txt"))
node._teardown(None, tmp_path)
assert not os.path.exists(os.path.join(tmp_path, "foo.txt"))
assert os.path.exists(os.path.join(destination, "foo.txt"))
def finalize(self):
"""Creates an AtomicCmd object based on the AtomicParam object. Once
finalized, the AtomicCmdBuilder cannot be modified further."""
if not self._object:
self._object = AtomicCmd(self.call, **self.kwargs)
return self._object
def test_pformat__atomiccmd__simple_with_temp_outfile():
cmd = AtomicCmd(("touch", "%(TEMP_OUT_RC)s"), TEMP_OUT_RC="bashrc")
assert pformat(cmd) == ("Command = touch '${TEMP_DIR}/bashrc'\n"
"STDOUT* = '${TEMP_DIR}/pipe_touch_%i.stdout'\n"
"STDERR* = '${TEMP_DIR}/pipe_touch_%i.stderr'") % (
id(cmd), id(cmd))
def test_pformat__atomiccmd__simple_with_temp_infile():
cmd = AtomicCmd(("cat", "%(TEMP_IN_FILE)s"), TEMP_IN_FILE="infile.txt")
assert pformat(cmd) == ("Command = cat '${TEMP_DIR}/infile.txt'\n"
"STDOUT* = '${TEMP_DIR}/pipe_cat_%i.stdout'\n"
"STDERR* = '${TEMP_DIR}/pipe_cat_%i.stderr'") % (
id(cmd), id(cmd))
def test_pformat__atomiccmd__simple_with_temp_stdin__set_cwd():
cmd = AtomicCmd("gzip", TEMP_IN_STDIN="stabstabstab", set_cwd=True)
assert pformat(cmd) == ("Command = gzip\n"
"STDIN* = 'stabstabstab'\n"
"STDOUT* = 'pipe_gzip_%i.stdout'\n"
"STDERR* = 'pipe_gzip_%i.stderr'\n"
"CWD = '${TEMP_DIR}'") % (id(cmd), id(cmd))
def __init__(self, input_prefix, output_prefix, nchroms, dependencies=()):
self._input_prefix = input_prefix
self._output_prefix = output_prefix
self._nchroms = nchroms
cmd = AtomicCmd(
("smartpca", "-p", "%(TEMP_OUT_PARAMS)s"),
TEMP_OUT_PARAMS="parameters.txt",
IN_FILE_BED=input_prefix + ".bed",
IN_FILE_BIM=input_prefix + ".bim",
IN_FILE_FAM=input_prefix + ".fam",
OUT_STDOUT=output_prefix + ".log",
OUT_EVEC=output_prefix + ".evec",
OUT_EVAL=output_prefix + ".eval",
OUT_SNPS=output_prefix + ".deleted_snps",
CHECK_VERSION=SMARTPCA_VERSION,
set_cwd=True,
)
CommandNode.__init__(
self,
description="<SmartPCA -> '%s.*>" % (output_prefix, ),
command=cmd,
dependencies=dependencies,
)
def __init__(self,
samples,
treefile,
bootstraps,
output_prefix,
dependencies=()):
rscript = rtools.rscript("zonkey", "tinytree.r")
cmd = AtomicCmd(("Rscript", rscript, "%(TEMP_OUT_FILE)s",
"%(IN_SAMPLES)s", "%(TEMP_OUT_PREFIX)s"),
AUX_RSCRIPT=rscript,
IN_SAMPLES=samples,
IN_FILE=treefile,
IN_BOOTSTRAPS=bootstraps,
TEMP_OUT_FILE="rerooted.newick",
TEMP_OUT_PREFIX=os.path.basename(output_prefix),
OUT_TREE_PDF=output_prefix + ".pdf",
OUT_TREE_PNG=output_prefix + ".png",
CHECK_RSCRIPT=RSCRIPT_VERSION,
CHECK_RSCRIPT_APE=rtools.requirement("ape"),
CHECK_RSCRIPT_GGPLOT2=rtools.requirement("ggplot2"),
CHECK_RSCRIPT_GRID=rtools.requirement("grid"))
self._treefile = treefile
self._bootstraps = bootstraps
CommandNode.__init__(self,
description="<DrawPhylogeny -> '%s.*'>" %
(output_prefix, ),
command=cmd,
dependencies=dependencies)
def __init__(self, config, reference, intervals, infiles, outfile,
dependencies=()):
self._basename = os.path.basename(outfile)
infiles = safe_coerce_to_tuple(infiles)
jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar")
command = AtomicJavaCmdBuilder(jar_file,
jre_options=config.jre_options)
command.set_option("-T", "IndelRealigner")
command.set_option("-R", "%(IN_REFERENCE)s")
command.set_option("-targetIntervals", "%(IN_INTERVALS)s")
command.set_option("-o", "%(OUT_BAMFILE)s")
command.set_option("--bam_compression", 0)
command.set_option("--disable_bam_indexing")
_set_input_files(command, infiles)
command.set_kwargs(IN_REFERENCE=reference,
IN_REF_DICT=fileutils.swap_ext(reference, ".dict"),
IN_INTERVALS=intervals,
OUT_BAMFILE=outfile,
CHECK_GATK=_get_gatk_version_check(config))
calmd = AtomicCmd(["samtools", "calmd", "-b",
"%(TEMP_IN_BAM)s", "%(IN_REF)s"],
TEMP_IN_BAM=self._basename,
IN_REF=reference,
TEMP_OUT_STDOUT=self._basename + ".calmd",
CHECK_VERSION=SAMTOOLS_VERSION)
description = "<GATK Indel Realigner (aligning): %s -> %r>" \
% (describe_files(infiles), outfile)
CommandNode.__init__(self,
description=description,
command=ParallelCmds([command.finalize(), calmd]),
dependencies=dependencies)
def __init__(self,
contigs,
mapping,
input_file,
output_prefix,
dependencies=()):
self._contigs = contigs
self._mapping = dict(zip(mapping.values(), mapping))
self._input_file = input_file
script = rtools.rscript("zonkey", "coverage.r")
cmd = AtomicCmd(
("Rscript", script, "%(TEMP_OUT_TABLE)s", "%(TEMP_OUT_PREFIX)s"),
AUX_RSCRIPT=script,
IN_FILE=input_file,
TEMP_OUT_TABLE="contigs.table",
OUT_PDF=output_prefix + ".pdf",
OUT_PNG=output_prefix + ".png",
TEMP_OUT_PREFIX=os.path.basename(output_prefix),
CHECK_R=RSCRIPT_VERSION,
CHECK_R_GGPLOT2=rtools.requirement("ggplot2"),
set_cwd=True,
)
CommandNode.__init__(
self,
description="<CoveragePlot -> '%s.*'>" % (output_prefix, ),
command=cmd,
dependencies=dependencies,
)
def __init__(self,
output_prefix,
tfam,
tped,
indep_filter=None,
indep_parameters=None,
plink_parameters=None,
dependencies=()):
temp_prefix = os.path.basename(output_prefix)
plink_cmd = [
"plink", "--make-bed", "--noweb", "--tped", "%(IN_TPED)s",
"--tfam", "%(IN_TFAM)s", "--out", "%(TEMP_OUT_PREFIX)s"
]
plink_cmd.extend(self._parse_parameters(plink_parameters))
command = AtomicCmd(plink_cmd,
IN_TPED=tped,
IN_TFAM=tfam,
TEMP_OUT_PREFIX=temp_prefix,
OUT_BED=output_prefix + ".bed",
OUT_BIM=output_prefix + ".bim",
OUT_FAM=output_prefix + ".fam",
OUT_LOG=output_prefix + ".log",
TEMP_OUT_NOSEX=temp_prefix + ".nosex",
TEMP_OUT_NOF=temp_prefix + ".nof",
CHECK_VERSION=PLINK_VERSION,
set_cwd=True)
CommandNode.__init__(self,
description="<BuildBEDFiles -> '%s.*'>" %
(output_prefix, ),
command=command,
dependencies=dependencies)
def __init__(self, infile, index_format=".bai", dependencies=()):
if index_format == ".bai":
samtools_call = ["samtools", "index", "%(IN_BAM)s", "%(OUT_IDX)s"]
elif index_format == ".csi":
samtools_call = [
"samtools", "index", "-c", "%(IN_BAM)s", "%(OUT_IDX)s"
]
else:
raise ValueError("Unknown format type %r; expected .bai or .csi" %
(index_format, ))
command = AtomicCmd(
samtools_call,
IN_BAM=infile,
OUT_IDX=infile + index_format,
CHECK_SAM=SAMTOOLS_VERSION,
)
CommandNode.__init__(
self,
description="<BAMIndex (%s): '%s'>" %
(index_format[1:].upper(), infile),
command=command,
dependencies=dependencies,
)
def __init__(self,
input_file,
output_prefix,
order,
samples,
dependencies=()):
self._samples = samples
self._order = tuple(order) + ("Sample", )
script = rtools.rscript("zonkey", "admixture.r")
cmd = AtomicCmd(("Rscript", script, "%(IN_FILE)s",
"%(TEMP_OUT_NAMES)s", "%(TEMP_OUT_PREFIX)s"),
AUX_RSCRIPT=script,
IN_FILE=input_file,
IN_SAMPLES=samples,
OUT_PDF=output_prefix + ".pdf",
OUT_PNG=output_prefix + ".png",
TEMP_OUT_NAMES="samples.txt",
TEMP_OUT_PREFIX=os.path.basename(output_prefix),
CHECK_R=RSCRIPT_VERSION,
CHECK_R_GGPLOT2=rtools.requirement("ggplot2"),
CHECK_R_RESHAPE2=rtools.requirement("reshape2"),
set_cwd=True)
CommandNode.__init__(self,
description="<AdmixturePlot -> '%s.*'>" %
(output_prefix, ),
command=cmd,
dependencies=dependencies)
def test_atomiccmd__commit_failure_cleanup(temp_folder):
counter = []
move_file = fileutils.move_file
def _monkey_move_file(source, destination):
if counter:
raise OSError("ARRRGHHH!")
counter.append(destination)
return move_file(source, destination)
destination, temp_folder = _setup_for_commit(temp_folder, False)
command = AtomicCmd(
("touch", "%(OUT_FILE_1)s", "%(OUT_FILE_2)s", "%(OUT_FILE_3)s"),
OUT_FILE_1=os.path.join(destination, "file_1"),
OUT_FILE_2=os.path.join(destination, "file_2"),
OUT_FILE_3=os.path.join(destination, "file_3"))
try:
fileutils.move_file = _monkey_move_file
command.run(temp_folder)
assert_equal(command.join(), [0])
assert_raises(OSError, command.commit, temp_folder)
assert_equal(tuple(os.listdir(destination)), ())
finally:
fileutils.move_file = move_file
def __init__(self, samples, prefix, output_prefix, dependencies=()):
abs_prefix = os.path.abspath(prefix)
script = rtools.rscript("zonkey", "pca.r")
call = [
"Rscript", script, abs_prefix, "%(IN_SAMPLES)s",
"%(TEMP_OUT_PREFIX)s"
]
cmd = AtomicCmd(call,
AUX_SCRIPT=script,
IN_FILE_EVAL=prefix + ".eval",
IN_FILE_EVEC=prefix + ".evec",
IN_SAMPLES=samples,
TEMP_OUT_PREFIX=os.path.basename(output_prefix),
OUT_PDF=output_prefix + ".pdf",
OUT_PNG=output_prefix + ".png",
CHECK_R=RSCRIPT_VERSION,
CHECK_R_GGPLOT2=rtools.requirement("ggplot2"),
CHECK_R_LABELS=rtools.requirement("ggrepel"),
set_cwd=True)
CommandNode.__init__(self,
description="<PlotPCA -> '%s.*'>" %
(output_prefix, ),
command=cmd,
dependencies=dependencies)
def test_atomiccmd__commit_failure_cleanup(tmp_path):
counter = []
move_file = fileutils.move_file
def _monkey_move_file(source, destination):
if counter:
raise OSError("ARRRGHHH!")
counter.append(destination)
return move_file(source, destination)
destination, tmp_path = _setup_for_commit(tmp_path, False)
command = AtomicCmd(
("touch", "%(OUT_FILE_1)s", "%(OUT_FILE_2)s", "%(OUT_FILE_3)s"),
OUT_FILE_1=os.path.join(destination, "file_1"),
OUT_FILE_2=os.path.join(destination, "file_2"),
OUT_FILE_3=os.path.join(destination, "file_3"),
)
try:
fileutils.move_file = _monkey_move_file
command.run(tmp_path)
assert command.join() == [0]
with pytest.raises(OSError):
command.commit(tmp_path)
assert tuple(os.listdir(destination)) == ()
finally:
fileutils.move_file = move_file
def test_atomiccmd__paths():
cmd = AtomicCmd(
"ls",
IN_AAA="/a/b/c",
IN_AAB="/x/y/z",
TEMP_IN_ABB="tmp_in",
OUT_AAA="/out/foo",
OUT_BBC="foo/bar",
TEMP_OUT_A="xyb",
EXEC_OTHER="true",
AUX_WAT="wat/wat",
CHECK_FUNC=bool,
OUT_STDERR="/var/log/pipe.stderr",
TEMP_OUT_STDOUT="pipe.stdout",
)
assert cmd.executables == frozenset(["ls", "true"])
assert cmd.requirements == frozenset([bool])
assert cmd.input_files == frozenset(["/a/b/c", "/x/y/z"])
assert cmd.output_files == frozenset(
["/out/foo", "foo/bar", "/var/log/pipe.stderr"])
assert cmd.auxiliary_files == frozenset(["wat/wat"])
assert cmd.expected_temp_files == frozenset(["foo", "bar", "pipe.stderr"])
assert "xyb" in cmd.optional_temp_files
assert "pipe.stdout" in cmd.optional_temp_files
def test_parallel_commands__join_before_run():
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.join = mock.join_1
cmd_2 = AtomicCmd(["ls"])
cmd_2.join = mock.join_2
cmd_3 = AtomicCmd(["ls"])
cmd_3.join = mock.join_3
cmds = ParallelCmds((cmd_3, cmd_2, cmd_1))
assert cmds.join() == [None, None, None]
assert mock.mock_calls == []
def test_pformat__atomiccmd__simple_with_temp_stdout__set_cwd():
cmd = AtomicCmd(("echo", "Room service. Room service."),
TEMP_OUT_STDOUT="pv",
set_cwd=True)
assert pformat(cmd) == ("Command = echo 'Room service. Room service.'\n"
"STDOUT* = 'pv'\n"
"STDERR* = 'pipe_echo_%i.stderr'\n"
"CWD = '${TEMP_DIR}'") % (id(cmd), )
def test_atomiccmd__pipes_out(tmp_path, stdout, stderr, kwargs):
call = ("bash", "-c",
"echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';")
cmd = AtomicCmd(call, **kwargs)
cmd.run(tmp_path)
assert cmd.join() == [0]
expected_files = []
for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
if tmpl is not None:
fname = tmpl.format(id(cmd))
result = (tmp_path / fname).read_text()
assert result == text
expected_files.append(fname)
assert set(os.listdir(tmp_path)) == set(expected_files)
def _test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
call = ("bash", "-c",
"echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';")
cmd = AtomicCmd(call, **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected_files = []
for (tmpl, text) in ((stdout, "STDOUT!"), (stderr, "STDERR!")):
if tmpl is not None:
fname = tmpl.format(id(cmd))
result = get_file_contents(os.path.join(temp_folder, fname))
assert_equal(result, text)
expected_files.append(fname)
assert_equal(set(os.listdir(temp_folder)), set(expected_files))
def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def test_atomiccmd__terminate_race_condition(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"],
TEMP_OUT_STDOUT=AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"],
TEMP_IN_STDIN=cmd_1,
OUT_STDOUT="piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def test_pformat__simple__running__set_cwd(temp_folder):
cmd = AtomicCmd(("sleep", "10"), set_cwd=True)
cmd.run(temp_folder)
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Running ...\n"
" STDOUT* = 'pipe_sleep_{id}.stdout'\n"
" STDERR* = 'pipe_sleep_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
cmd.terminate()
cmd.join()
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"),
TEMP_OUT_STDOUT="bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def test_pformat__simple__running(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
assert_equal(pformat(cmd),
("Command = sleep 10\n"
"Status = Running ...\n"
"STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
"STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
"CWD = '{cwd}'").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
cmd.terminate()
cmd.join()
def _do_test_atomiccmd__terminate(temp_folder, raise_on_terminate):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
killpg_was_called = []
def _wrap_killpg(pid, sig):
assert_equal(pid, cmd._proc.pid)
assert_equal(sig, signal.SIGTERM)
killpg_was_called.append(True)
if raise_on_terminate:
raise OSError("KABOOM!")
with Monkeypatch("os.killpg", _wrap_killpg):
cmd.terminate()
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert killpg_was_called
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd=False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT=os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO="bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_pformat__simple__killed(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Terminated with signal SIGTERM\n"
" STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id=id(cmd),
temp_dir=temp_folder,
cwd=os.getcwd()))