def test_sequential_commands__abort_on_error_1(temp_folder):
cmd_1 = AtomicCmd("false")
cmd_2 = AtomicCmd(("sleep", 10))
cmd_3 = AtomicCmd(("sleep", 10))
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(temp_folder)
assert_equal(cmds.join(), [1, None, None])
def _do_test(cls):
cmd_mock_1 = AtomicCmd(("true", ),
CHECK_A=id,
EXEC_1="false",
IN_1="/foo/bar/in_1.file",
IN_2="/foo/bar/in_2.file",
OUT_1="/bar/foo/out",
TEMP_OUT_1="out.log",
AUX_A="/aux/fA",
AUX_B="/aux/fB")
cmd_mock_2 = AtomicCmd(("false", ),
CHECK_A=list,
EXEC_1="echo",
EXEC_2="java",
IN_1="/foo/bar/in.file",
OUT_1="out.txt")
obj = cls([cmd_mock_1, cmd_mock_2])
assert_equal(obj.executables,
cmd_mock_1.executables | cmd_mock_2.executables)
assert_equal(obj.requirements,
cmd_mock_1.requirements | cmd_mock_2.requirements)
assert_equal(obj.input_files,
cmd_mock_1.input_files | cmd_mock_2.input_files)
assert_equal(obj.output_files,
cmd_mock_1.output_files | cmd_mock_2.output_files)
assert_equal(obj.auxiliary_files,
cmd_mock_1.auxiliary_files | cmd_mock_2.auxiliary_files)
assert_equal(obj.expected_temp_files, frozenset(["out", "out.txt"]))
assert_equal(
obj.optional_temp_files,
cmd_mock_1.optional_temp_files | cmd_mock_2.optional_temp_files)
def __init__(self, config, reference, input_files, output_file,
dependencies):
cat_cmds, cat_obj = concatenate_input_bams(config, input_files)
cmd_map = AtomicCmd([
"mapDamage", "-n", _MAPDAMAGE_MAX_READS, "-i", "-", "-d",
"%(TEMP_DIR)s", "-r", reference
],
IN_STDIN=cat_obj,
CHECK_VERSION=MAPDAMAGE_VERSION)
train_cmds = ParallelCmds(cat_cmds + [cmd_map])
cat_cmds, cat_obj = concatenate_input_bams(config, input_files)
cmd_scale = AtomicCmd([
"mapDamage", "--rescale-only", "-n", _MAPDAMAGE_MAX_READS, "-i",
"-", "-d", "%(TEMP_DIR)s", "-r", reference, "--rescale-out",
"%(OUT_BAM)s"
],
IN_STDIN=cat_obj,
OUT_BAM=output_file,
CHECK_VERSION=MAPDAMAGE_VERSION)
rescale_cmds = ParallelCmds(cat_cmds + [cmd_scale])
description = "<mapDamageRescale: %i file(s) -> '%s'>" % (
len(input_files), output_file)
CommandNode.__init__(self,
command=SequentialCmds([train_cmds,
rescale_cmds]),
description=description,
dependencies=dependencies)
def test_atomiccmd__commit_before_join(temp_folder):
cmd = AtomicCmd(("sleep", "0.1"))
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.join()
def test_atomiccmd__paths_non_str(temp_folder):
cmd = AtomicCmd(("touch", 1234),
OUT_FOO = "1234",
set_cwd = True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "1234"))
def test_atomiccmd__paths__key(temp_folder):
cmd = AtomicCmd(("echo", "-n", "%(TEMP_DIR)s"),
OUT_STDOUT=AtomicCmd.PIPE)
cmd.run(temp_folder)
path = cmd._proc.stdout.read()
assert os.path.samefile(temp_folder, path), (temp_folder, path)
assert_equal(cmd.join(), [0])
def _do_test_parallel_commands__ready_two(first, second, result):
cmd_mock_1 = flexmock(AtomicCmd(["ls"]))
cmd_mock_1.should_receive('ready').and_return(first).at_least.once
cmd_mock_2 = flexmock(AtomicCmd(["ls"]))
cmd_mock_2.should_receive('ready').and_return(second)
cmds = ParallelCmds([cmd_mock_1, cmd_mock_2])
assert_equal(cmds.ready(), result)
def test_sequential_commands__abort_on_error_3(temp_folder):
cmd_1 = AtomicCmd("true")
cmd_2 = AtomicCmd("true")
cmd_3 = AtomicCmd("false")
cmds = SequentialCmds([cmd_1, cmd_2, cmd_3])
cmds.run(temp_folder)
assert_equal(cmds.join(), [0, 0, 1])
def _do_test_atomiccmd__pipes_out(temp_folder, stdout, stderr, kwargs):
cmd = AtomicCmd(("bash", "-c", "echo -n 'STDERR!' > /dev/stderr; echo -n 'STDOUT!';"), **kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result_out = get_file_contents(os.path.join(temp_folder, stdout.format(id(cmd))))
result_err = get_file_contents(os.path.join(temp_folder, stderr.format(id(cmd))))
assert_equal(result_out, "STDOUT!")
assert_equal(result_err, "STDERR!")
def test_pformat__atomiccmd__simple_with_stdin__cmd():
cmd_1 = AtomicCmd("gzip", OUT_STDOUT=AtomicCmd.PIPE)
cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1)
assert_equal(pformat(cmd_2),
("<Command = ['gzip']\n"
" STDIN = <PIPE>\n"
" STDOUT* = '${TEMP_DIR}/pipe_gzip_%i.stdout'\n"
" STDERR* = '${TEMP_DIR}/pipe_gzip_%i.stderr'>") %
(id(cmd_2), id(cmd_2)))
def test_atomiccmd__pipes_stdin(temp_folder):
fname = os.path.join("tests", "data", "fasta_file.fasta")
cmd = AtomicCmd("cat", IN_STDIN=fname, OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result,
">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_atomiccmd__commit_missing_files(temp_folder):
destination, temp_folder = _setup_for_commit(temp_folder, False)
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO = os.path.join(destination, "1234"),
OUT_BAR = os.path.join(destination, "4567"))
cmd.run(temp_folder)
cmd.join()
before = set(os.listdir(temp_folder))
assert_raises(CmdError, cmd.commit, temp_folder)
assert_equal(before, set(os.listdir(temp_folder)))
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN = "infile.fasta",
OUT_STDOUT = "result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "infile.fasta"))
assert_equal(result, "a\nbc\nd")
def test_atomiccmd__pipes_stdin(temp_folder):
fname = os.path.join("tests", "data", "fasta_file.fasta")
cmd = AtomicCmd("cat",
IN_STDIN = fname,
OUT_STDOUT = "result.txt")
assert_equal(cmd.input_files, frozenset([fname]))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(result, ">This_is_FASTA!\nACGTN\n>This_is_ALSO_FASTA!\nCGTNA\n")
def test_pformat__simple__done__set_cwd(temp_folder):
cmd = AtomicCmd("true", set_cwd = True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd), ("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = 'pipe_true_{id}.stdout'\n"
" STDERR* = 'pipe_true_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id = id(cmd),
temp_dir = temp_folder))
def test_atomiccmd__pipes_stdin__temp_file(temp_folder):
cmd = AtomicCmd("cat",
TEMP_IN_STDIN="infile.fasta",
OUT_STDOUT="result.txt")
assert_equal(cmd.input_files, frozenset())
set_file_contents(os.path.join(temp_folder, "infile.fasta"), "a\nbc\nd")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "infile.fasta"))
assert_equal(result, "a\nbc\nd")
def test_pformat__simple__done__set_cwd(temp_folder):
cmd = AtomicCmd("true", set_cwd=True)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(pformat(cmd),
("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = 'pipe_true_{id}.stdout'\n"
" STDERR* = 'pipe_true_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id=id(cmd),
temp_dir=temp_folder))
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT="result.txt",
set_cwd=set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def _do_test_atomiccmd__paths_temp_in(temp_folder, set_cwd, kwargs):
cmd = AtomicCmd(("echo", "-n", "%%(%s)s" % tuple(kwargs.keys())),
TEMP_OUT_STDOUT = "result.txt",
set_cwd = set_cwd,
**kwargs)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
expected = os.path.join("" if set_cwd else temp_folder, "test_file")
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert_equal(os.path.abspath(expected), os.path.abspath(result))
def _setup_mocks_for_failure(*do_mocks):
results = []
for do_mock in do_mocks:
if do_mock:
mock = flexmock(AtomicCmd(("sleep", 10)))
mock.should_receive('terminate')
mock.should_receive('join').and_return(['SIGTERM'])
else:
mock = AtomicCmd("false")
results.append(mock)
return results
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"], TEMP_OUT_STDOUT=AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"], TEMP_IN_STDIN=cmd_1, OUT_STDOUT="piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
assert_equal(pypeline.atomiccmd.command._PROCS, set())
cmd = AtomicCmd("ls")
cmd.run(temp_folder)
ref = iter(pypeline.atomiccmd.command._PROCS).next()
assert ref
assert_equal(ref(), cmd._proc)
assert_equal(cmd.join(), [0])
cmd = func(cmd, temp_folder)
assert ref not in pypeline.atomiccmd.command._PROCS
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
cmd._proc.wait() # pylint: disable=W0212
assert_equal(pformat(cmd), ("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id = id(cmd),
cwd = os.getcwd(),
temp_dir = temp_folder))
assert_equal(cmd.join(), [0])
def test_commandnode_teardown__missing_optional_files(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
TEMP_OUT_BAR = "bar.txt",
OUT_STDOUT = os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
node._teardown(None, temp_folder)
assert_equal(os.listdir(temp_folder), [])
assert_equal(os.listdir(destination), ["foo.txt"])
def _do_test_atomiccmd__set_cwd(temp_folder, set_cwd):
cwd = os.getcwd()
cmd = AtomicCmd(("bash", "-c", "echo -n ${PWD}"),
TEMP_OUT_STDOUT = "result.txt",
set_cwd = set_cwd)
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert_equal(cwd, os.getcwd())
expected = temp_folder if set_cwd else cwd
result = get_file_contents(os.path.join(temp_folder, "result.txt"))
assert os.path.samefile(expected, result), "%r != %r" % (expected, result)
def _do_test_atomiccmd__cleanup_proc(temp_folder, func):
assert_equal(pypeline.atomiccmd.command._PROCS, set())
cmd = AtomicCmd("ls")
cmd.run(temp_folder)
ref = iter(pypeline.atomiccmd.command._PROCS).next()
assert ref
assert_equal(ref(), cmd._proc)
assert_equal(cmd.join(), [0])
cmd = func(cmd, temp_folder)
assert ref not in pypeline.atomiccmd.command._PROCS
def test_pformat__simple__done__before_join(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
cmd._proc.wait() # pylint: disable=W0212
assert_equal(pformat(cmd),
("<Command = ['true']\n"
" Status = Exited with return-code 0\n"
" STDOUT* = '{temp_dir}/pipe_true_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_true_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id=id(cmd),
cwd=os.getcwd(),
temp_dir=temp_folder))
assert_equal(cmd.join(), [0])
def test_commandnode_teardown(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
OUT_STDOUT = os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
assert os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert not os.path.exists(os.path.join(destination, "foo.txt"))
node._teardown(None, temp_folder)
assert not os.path.exists(os.path.join(temp_folder, "foo.txt"))
assert os.path.exists(os.path.join(destination, "foo.txt"))
def test_atomiccmd__piping_is_only_allowed_once(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "foo\nbar"],
OUT_STDOUT=AtomicCmd.PIPE)
cmd_2a = AtomicCmd(["grep", "foo"],
IN_STDIN=cmd_1)
cmd_2b = AtomicCmd(["grep", "bar"],
IN_STDIN=cmd_1)
cmd_1.run(temp_folder)
cmd_2a.run(temp_folder)
assert_raises(CmdError, cmd_2b.run, temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2a.join(), [0])
assert_equal(cmd_2b.join(), [None])
def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def _do_test_pformat__sets__simple(cls, description):
cmd_1 = AtomicCmd(("echo", "foo"), OUT_STDOUT=AtomicCmd.PIPE)
cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1)
cmd = cls((cmd_1, cmd_2))
assert_equal(pformat(cmd), ("<{description}:\n"
" - <00> Command = ['echo', 'foo']\n"
" STDOUT = <01>\n"
" STDERR* = '${{TEMP_DIR}}/pipe_echo_{cmd_1_id}.stderr'\n"
" - <01> Command = ['gzip']\n"
" STDIN = <00>\n"
" STDOUT* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2_id}.stdout'\n"
" STDERR* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2_id}.stderr'>") \
.format(description = description,
cmd_1_id = id(cmd_1),
cmd_2_id = id(cmd_2)))
def test_commandnode_teardown__extra_files_in_temp(temp_folder):
destination, temp_folder = _setup_temp_folders(temp_folder)
cmd = AtomicCmd(("echo", "-n", "1 2 3"),
OUT_STDOUT = os.path.join(destination, "foo.txt"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
node = CommandNode(cmd)
set_file_contents(os.path.join(temp_folder, "bar.txt"), "1 2 3")
temp_files_before = set(os.listdir(temp_folder))
dest_files_before = set(os.listdir(destination))
assert_raises(CmdNodeError, node._teardown, None, temp_folder)
assert_equal(temp_files_before, set(os.listdir(temp_folder)))
assert_equal(dest_files_before, set(os.listdir(destination)))
def _setup_for_commit(temp_folder, create_cmd = True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO = os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd
def _setup_for_commit(temp_folder, create_cmd=True):
destination = os.path.join(temp_folder, "out")
temp_folder = os.path.join(temp_folder, "tmp")
os.makedirs(destination)
os.makedirs(temp_folder)
if not create_cmd:
return destination, temp_folder
cmd = AtomicCmd(("touch", "%(OUT_FOO)s"),
OUT_FOO=os.path.join(destination, "1234"))
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
return destination, temp_folder, cmd
def __init__(self, config, d_make, bedn, mappa, unique, dependencies=()):
inbedfile = d_make.bedfiles[bedn]
basename, extension = os.path.splitext(os.path.basename(inbedfile))
bname = "{}_MappaOnly{}".format(basename, extension)
dest = os.path.join(config.temp_local, bname)
d_make.bedfiles[bedn] = dest
call1 = [
"python",
os.path.join(PREFIX, "intersectmappabed.py"), "%(IN_BED)s",
"%(IN_MAPPA)s",
str(unique), "%(OUT_DEST)s"
]
cmd = AtomicCmd(call1,
IN_BED=inbedfile,
IN_MAPPA=mappa,
OUT_DEST=dest,
CHECK_VERSION=PYTHON_VERSION)
description = ("<CLEANBEDFILES: '%s' -> '%s', Uniqueness: '%s'>" %
(inbedfile, dest, unique))
CommandNode.__init__(self,
description=description,
command=cmd,
dependencies=dependencies)
def __init__(self,
infile,
outfile,
genome,
from_start=0,
from_end=0,
strand_relative=False,
dependencies=()):
if type(from_start) != type(from_end):
raise ValueError(
"'from_start' and 'from_end' should be of same type!")
call = [
"slopBed", "-i", "%(IN_FILE)s", "-g", "%(IN_GENOME)s", "-l",
str(from_start), "-r",
str(from_end)
]
if strand_relative:
call.append("-s")
if type(from_start) is float:
call.append("-pct")
command = AtomicCmd(call,
IN_FILE=infile,
IN_GENOME=genome,
OUT_STDOUT=outfile)
description = "<SlopBed: '%s' -> '%s'>" % (infile, outfile)
CommandNode.__init__(self,
description=description,
command=command,
dependencies=dependencies)
def __init__(self, config, reference, input_files, output_directory,
dependencies):
cat_cmds, cat_obj = concatenate_input_bams(config, input_files)
cmd_map = AtomicCmd(
[
"mapDamage", "--no-stats", "-n", _MAPDAMAGE_MAX_READS, "-i",
"-", "-d", "%(TEMP_DIR)s", "-r", reference
],
IN_STDIN=cat_obj,
OUT_FREQ_3p=os.path.join(output_directory, "3pGtoA_freq.txt"),
OUT_FREQ_5p=os.path.join(output_directory, "5pCtoT_freq.txt"),
OUT_COMP_USER=os.path.join(output_directory, "dnacomp.txt"),
OUT_PLOT_FRAG=os.path.join(output_directory,
"Fragmisincorporation_plot.pdf"),
OUT_PLOT_LEN=os.path.join(output_directory, "Length_plot.pdf"),
OUT_LENGTH=os.path.join(output_directory, "lgdistribution.txt"),
OUT_MISINCORP=os.path.join(output_directory,
"misincorporation.txt"),
OUT_LOG=os.path.join(output_directory, "Runtime_log.txt"),
CHECK_VERSION=MAPDAMAGE_VERSION)
description = "<mapDamage: %i file(s) -> '%s'>" % (len(input_files),
output_directory)
CommandNode.__init__(self,
command=ParallelCmds(cat_cmds + [cmd_map]),
description=description,
dependencies=dependencies)
def __init__(self, config, reference, intervals, infiles, outfile,
dependencies=()):
self._basename = os.path.basename(outfile)
infiles = safe_coerce_to_tuple(infiles)
jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar")
command = AtomicJavaCmdBuilder(jar_file,
jre_options=config.jre_options)
command.set_option("-T", "IndelRealigner")
command.set_option("-R", "%(IN_REFERENCE)s")
command.set_option("-targetIntervals", "%(IN_INTERVALS)s")
command.set_option("-o", "%(OUT_BAMFILE)s")
command.set_option("--bam_compression", 0)
command.set_option("--disable_bam_indexing")
_set_input_files(command, infiles)
command.set_kwargs(IN_REFERENCE=reference,
IN_REF_DICT=fileutils.swap_ext(reference, ".dict"),
IN_INTERVALS=intervals,
OUT_BAMFILE=outfile,
CHECK_GATK=_get_gatk_version_check(config))
calmd = AtomicCmd(["samtools", "calmd", "-b",
"%(TEMP_IN_BAM)s", "%(IN_REF)s"],
TEMP_IN_BAM=self._basename,
IN_REF=reference,
TEMP_OUT_STDOUT=self._basename + ".calmd",
CHECK_VERSION=SAMTOOLS_VERSION)
description = "<Indel Realigner (aligning): %s -> %r>" \
% (describe_files(infiles), outfile)
CommandNode.__init__(self,
description=description,
command=ParallelCmds([command.finalize(), calmd]),
dependencies=dependencies)
def finalize(self):
"""Creates an AtomicCmd object based on the AtomicParam object. Once
finalized, the AtomicCmdBuilder cannot be modified further."""
if not self._object:
self._object = AtomicCmd(self.call, **self.kwargs)
return self._object
def test_atomicmcd__exec__reqobj():
reqobj = RequirementObj(call=("echo", "version"),
search="version",
pprint="{0}",
checks=str)
cmd = AtomicCmd("true", CHECK_VERSION=reqobj)
assert_equal(cmd.requirements, frozenset([reqobj]))
def test_atomiccmd__ready(temp_folder):
cmd = AtomicCmd("ls")
assert_equal(cmd.join(), [None])
assert not cmd.ready()
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert cmd.ready()
def _do_test_atomiccmd__stdout(temp_folder, kwargs):
cmd = AtomicCmd(("echo", "-n", "foo"), **kwargs)
assert cmd.stdout is None
cmd.run(temp_folder)
assert isinstance(cmd.stdout, types.FileType), cmd.stdout
assert_equal(cmd.stdout.read(), "foo")
cmd.join()
cmd.commit(temp_folder)
assert cmd.stdout is None
def test_atomiccmd__terminate_race_condition(temp_folder):
cmd = AtomicCmd("true")
cmd.run(temp_folder)
while cmd._proc.poll() is None:
pass
cmd.terminate()
assert_equal(cmd.join(), [0])
def test_atomiccmd__piping_temp(temp_folder):
cmd_1 = AtomicCmd(["echo", "-n", "#@!$^"],
TEMP_OUT_STDOUT = AtomicCmd.PIPE)
assert_equal(cmd_1.output_files, frozenset())
cmd_2 = AtomicCmd(["cat"],
TEMP_IN_STDIN = cmd_1,
OUT_STDOUT = "piped.txt")
assert_equal(cmd_2.input_files, frozenset())
cmd_1.run(temp_folder)
cmd_2.run(temp_folder)
assert_equal(cmd_1.join(), [0])
assert_equal(cmd_2.join(), [0])
result = get_file_contents(os.path.join(temp_folder, "piped.txt"))
assert_equal(result, "#@!$^")
def test_pformat__simple__running__set_cwd(temp_folder):
cmd = AtomicCmd(("sleep", "10"), set_cwd = True)
cmd.run(temp_folder)
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Running ...\n"
" STDOUT* = 'pipe_sleep_{id}.stdout'\n"
" STDERR* = 'pipe_sleep_{id}.stderr'\n"
" CWD = '{temp_dir}'>").format(id = id(cmd),
temp_dir = temp_folder))
cmd.terminate()
cmd.join()
def test_atomiccmd__commit_temp_only(temp_folder):
cmd = AtomicCmd(("echo", "foo"),
TEMP_OUT_STDOUT = "bar.txt")
cmd.run(temp_folder)
assert_equal(cmd.join(), [0])
assert os.path.exists(os.path.join(temp_folder, "bar.txt"))
cmd.commit(temp_folder)
assert_equal(os.listdir(temp_folder), [])
def _do_test_atomiccmd__terminate(temp_folder, raise_on_terminate):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
killpg_was_called = []
def _wrap_killpg(pid, sig):
assert_equal(pid, cmd._proc.pid)
assert_equal(sig, signal.SIGTERM)
killpg_was_called.append(True)
if raise_on_terminate:
raise OSError("KABOOM!")
with Monkeypatch("os.killpg", _wrap_killpg):
cmd.terminate()
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert killpg_was_called
def test_atomiccmd__commit_temp_out(temp_folder):
dest, temp = _setup_for_commit(temp_folder, create_cmd = False)
cmd = AtomicCmd(("echo", "foo"),
OUT_STDOUT = os.path.join(dest, "foo.txt"),
TEMP_OUT_FOO = "bar.txt")
cmd.run(temp)
assert_equal(cmd.join(), [0])
set_file_contents(os.path.join(temp, "bar.txt"), "1 2 3")
cmd.commit(temp)
assert_equal(os.listdir(temp), [])
assert_equal(os.listdir(dest), ["foo.txt"])
def test_pformat__simple__killed(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
assert_equal(pformat(cmd), ("<Command = ['sleep', '10']\n"
" Status = Terminated with signal SIGTERM\n"
" STDOUT* = '{temp_dir}/pipe_sleep_{id}.stdout'\n"
" STDERR* = '{temp_dir}/pipe_sleep_{id}.stderr'\n"
" CWD = '{cwd}'>").format(id = id(cmd),
temp_dir = temp_folder,
cwd = os.getcwd()))
def test_atomiccmd__terminate_sigkill(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd._proc.kill()
assert_equal(cmd.join(), ["SIGKILL"])
def test_atomiccmd__commit_while_running(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
assert_raises(CmdError, cmd.commit, temp_folder)
cmd.terminate()
cmd.join()
def commit(self, temp):
AtomicCmd.commit(self, temp)
os.remove(os.path.join(destination, "foo.txt"))
def test_atomiccmd__terminate_sigterm(temp_folder):
cmd = AtomicCmd(("sleep", "10"))
cmd.run(temp_folder)
cmd.terminate()
assert_equal(cmd.join(), ["SIGTERM"])
