def _do_test_parallel_commands__ready_two(first, second, result):
cmd_mock_1 = flexmock(AtomicCmd(["ls"]))
cmd_mock_1.should_receive('ready').and_return(first).at_least.once
cmd_mock_2 = flexmock(AtomicCmd(["ls"]))
cmd_mock_2.should_receive('ready').and_return(second)
cmds = ParallelCmds([cmd_mock_1, cmd_mock_2])
assert_equal(cmds.ready(), result)
def _do_test_parallel_commands__ready_two(first, second, result):
cmd_mock_1 = flexmock(AtomicCmd(["ls"]))
cmd_mock_1.should_receive('ready').and_return(first).at_least.once
cmd_mock_2 = flexmock(AtomicCmd(["ls"]))
cmd_mock_2.should_receive('ready').and_return(second)
cmds = ParallelCmds([cmd_mock_1, cmd_mock_2])
assert_equal(cmds.ready(), result)
def test_parallel_commands__join_before_run():
mocks = []
for value in reversed(range(3)):
cmd_mock = flexmock(AtomicCmd("true"))
cmd_mock.should_receive('join').and_return([value]).never
mocks.append(cmd_mock)
cmds = ParallelCmds(mocks)
assert_equal(cmds.join(), [None, None, None])
def test_parallel_commands__ready_single(value):
cmd = AtomicCmd(["ls"])
cmd.ready = Mock()
cmd.ready.return_value = value
cmds = ParallelCmds([cmd])
assert cmds.ready() == value
cmd.ready.assert_called()
def test_parallel_commands__join_before_run():
mocks = []
for value in reversed(range(3)):
cmd_mock = flexmock(AtomicCmd("true"))
cmd_mock.should_receive('join').and_return([value]).never
mocks.append(cmd_mock)
cmds = ParallelCmds(mocks)
assert_equal(cmds.join(), [None, None, None])
def test_parallel_commands__run():
mocks = []
for _ in range(3):
cmd_mock = flexmock(AtomicCmd(["ls"]))
cmd_mock.should_receive('run').with_args("xTMPx").once
mocks.append(cmd_mock)
cmds = ParallelCmds(mocks)
cmds.run("xTMPx")
def test_parallel_commands__run():
mocks = []
for _ in range(3):
cmd_mock = flexmock(AtomicCmd(["ls"]))
cmd_mock.should_receive('run').with_args("xTMPx").once
mocks.append(cmd_mock)
cmds = ParallelCmds(mocks)
cmds.run("xTMPx")
def test_parallel_commands__join_before_run():
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.join = mock.join_1
cmd_2 = AtomicCmd(["ls"])
cmd_2.join = mock.join_2
cmd_3 = AtomicCmd(["ls"])
cmd_3.join = mock.join_3
cmds = ParallelCmds((cmd_3, cmd_2, cmd_1))
assert cmds.join() == [None, None, None]
assert mock.mock_calls == []
def test_parallel_commands__ready_two(first, second, result):
cmd_1 = AtomicCmd(["ls"])
cmd_1.ready = Mock()
cmd_1.ready.return_value = first
cmd_2 = AtomicCmd(["ls"])
cmd_2.ready = Mock()
cmd_2.ready.return_value = second
cmds = ParallelCmds([cmd_1, cmd_2])
assert cmds.ready() == result
cmd_1.ready.assert_called()
assert bool(first) == bool(cmd_2.ready.call_count)
def __init__(self, infile, outfile, regions, options, dependencies=()):
vcffilter = factory.new("vcf_filter")
vcffilter.add_value("%(IN_VCF)s")
for contig in regions["HomozygousContigs"]:
vcffilter.add_option("--homozygous-chromosome", contig)
vcffilter.set_kwargs(IN_VCF=infile, OUT_STDOUT=AtomicCmd.PIPE)
apply_options(vcffilter, options)
bgzip = AtomicCmdBuilder(["bgzip"],
IN_STDIN=vcffilter,
OUT_STDOUT=outfile)
description = "<VCFFilter: '%s' -> '%s'>" % (
infile,
outfile,
)
CommandNode.__init__(
self,
description=description,
command=ParallelCmds([vcffilter.finalize(),
bgzip.finalize()]),
dependencies=dependencies,
)
def __init__(self, config, reference, intervals, infiles, outfile,
dependencies=()):
self._basename = os.path.basename(outfile)
infiles = safe_coerce_to_tuple(infiles)
jar_file = os.path.join(config.jar_root, "GenomeAnalysisTK.jar")
command = AtomicJavaCmdBuilder(jar_file,
jre_options=config.jre_options)
command.set_option("-T", "IndelRealigner")
command.set_option("-R", "%(IN_REFERENCE)s")
command.set_option("-targetIntervals", "%(IN_INTERVALS)s")
command.set_option("-o", "%(OUT_BAMFILE)s")
command.set_option("--bam_compression", 0)
command.set_option("--disable_bam_indexing")
_set_input_files(command, infiles)
command.set_kwargs(IN_REFERENCE=reference,
IN_REF_DICT=fileutils.swap_ext(reference, ".dict"),
IN_INTERVALS=intervals,
OUT_BAMFILE=outfile,
CHECK_GATK=_get_gatk_version_check(config))
calmd = AtomicCmd(["samtools", "calmd", "-b",
"%(TEMP_IN_BAM)s", "%(IN_REF)s"],
TEMP_IN_BAM=self._basename,
IN_REF=reference,
TEMP_OUT_STDOUT=self._basename + ".calmd",
CHECK_VERSION=SAMTOOLS_VERSION)
description = "<GATK Indel Realigner (aligning): %s -> %r>" \
% (describe_files(infiles), outfile)
CommandNode.__init__(self,
description=description,
command=ParallelCmds([command.finalize(), calmd]),
dependencies=dependencies)
def __init__(
self,
input_file_1,
output_file,
reference,
prefix,
input_file_2=None,
threads=1,
algorithm="mem",
mapping_options={},
cleanup_options={},
dependencies=(),
):
if algorithm not in ("mem", "bwasw"):
raise NotImplementedError("BWA algorithm %r not implemented" %
(algorithm, ))
threads = _get_max_threads(reference, threads)
aln = _new_bwa_command(
("bwa", algorithm, prefix, "%(IN_FILE_1)s"),
prefix,
IN_FILE_1=input_file_1,
OUT_STDOUT=AtomicCmd.PIPE,
)
if input_file_2:
aln.add_value("%(IN_FILE_2)s")
aln.set_kwargs(IN_FILE_2=input_file_2)
aln.set_option("-t", threads)
# Mark alternative hits as secondary; required by e.g. Picard
aln.set_option("-M")
cleanup = _new_cleanup_command(aln,
output_file,
reference,
paired_end=input_file_1
and input_file_2)
apply_options(aln, mapping_options)
apply_options(cleanup, cleanup_options)
description = _get_node_description(
name="BWA",
algorithm="%s%s" %
(algorithm.upper(), "_PE" if input_file_2 else "_SE"),
input_files_1=input_file_1,
input_files_2=input_file_2,
prefix=prefix,
)
CommandNode.__init__(
self,
command=ParallelCmds([aln.finalize(),
cleanup.finalize()]),
description=description,
threads=threads,
dependencies=dependencies,
)
def test_pformat__sets__nested():
cmd_1 = AtomicCmd(("echo", "foo"), OUT_STDOUT=AtomicCmd.PIPE)
cmd_2 = AtomicCmd("gzip", IN_STDIN=cmd_1)
cmd_3 = AtomicCmd("sha1sum")
set_1 = ParallelCmds((cmd_1, cmd_2))
set_2 = SequentialCmds((set_1, cmd_3))
assert pformat(set_2) == (
"Sequential processes:\n"
" Parallel processes:\n"
" Process 1:\n"
" Command = echo foo\n"
" STDOUT = Piped to process 2\n"
" STDERR* = '${{TEMP_DIR}}/pipe_echo_{cmd_1}.stderr'\n"
"\n"
" Process 2:\n"
" Command = gzip\n"
" STDIN = Piped from process 1\n"
" STDOUT* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2}.stdout'\n"
" STDERR* = '${{TEMP_DIR}}/pipe_gzip_{cmd_2}.stderr'\n"
"\n"
" Process 3:\n"
" Command = sha1sum\n"
" STDOUT* = '${{TEMP_DIR}}/pipe_sha1sum_{cmd_3}.stdout'\n"
" STDERR* = '${{TEMP_DIR}}/pipe_sha1sum_{cmd_3}.stderr'").format(
cmd_1=id(cmd_1), cmd_2=id(cmd_2), cmd_3=id(cmd_3))
def test_parallel_commands__run():
mock = Mock()
cmd_1 = AtomicCmd(["ls"])
cmd_1.run = mock.run_1
cmd_2 = AtomicCmd(["ls"])
cmd_2.run = mock.run_2
cmd_3 = AtomicCmd(["ls"])
cmd_3.run = mock.run_3
cmds = ParallelCmds((cmd_1, cmd_2, cmd_3))
cmds.run("xTMPx")
assert mock.mock_calls == [
call.run_1("xTMPx"),
call.run_2("xTMPx"),
call.run_3("xTMPx"),
]
def __init__(
self,
reference,
input_files,
output_directory,
title="mapDamage",
options={},
dependencies=(),
):
merge = merge_bam_files_command(input_files)
command = AtomicCmdBuilder(
[
"mapDamage",
"--no-stats",
# Prevent references with many contigs from using excessive
# amounts of memory, at the cost of per-contig statistics:
"--merge-reference-sequences",
"-t",
title,
"-i",
"-",
"-d",
"%(TEMP_DIR)s",
"-r",
"%(IN_REFERENCE)s",
],
IN_STDIN=merge,
IN_REFERENCE=reference,
OUT_FREQ_3p=os.path.join(output_directory, "3pGtoA_freq.txt"),
OUT_FREQ_5p=os.path.join(output_directory, "5pCtoT_freq.txt"),
OUT_COMP_USER=os.path.join(output_directory, "dnacomp.txt"),
OUT_PLOT_FRAG=os.path.join(
output_directory, "Fragmisincorporation_plot.pdf"
),
OUT_PLOT_LEN=os.path.join(output_directory, "Length_plot.pdf"),
OUT_LENGTH=os.path.join(output_directory, "lgdistribution.txt"),
OUT_MISINCORP=os.path.join(output_directory, "misincorporation.txt"),
OUT_LOG=os.path.join(output_directory, "Runtime_log.txt"),
TEMP_OUT_STDOUT="pipe_mapDamage.stdout",
TEMP_OUT_STDERR="pipe_mapDamage.stderr",
CHECK_RSCRIPT=RSCRIPT_VERSION,
CHECK_MAPDAMAGE=MAPDAMAGE_VERSION,
)
apply_options(command, options)
CommandNode.__init__(
self,
command=ParallelCmds([merge, command.finalize()]),
description="<mapDamage (plots): %s -> '%s'>"
% (describe_files(merge.input_files), output_directory,),
dependencies=dependencies,
)
def __init__(self, parameters):
commands = [
parameters.commands[key].finalize()
for key in ("cat", "filter", "bgzip")
]
description = "<VCFFilter: '%s' -> '%s'>" % (parameters.infile,
parameters.outfile)
CommandNode.__init__(self,
description=description,
command=ParallelCmds(commands),
dependencies=parameters.dependencies)
def __init__(
self,
reference,
infile,
bedfile,
outfile,
mpileup_options={},
bcftools_options={},
dependencies=(),
):
mpileup = AtomicCmdBuilder(
("bcftools", "mpileup", "%(IN_BAMFILE)s"),
IN_BAMFILE=infile,
IN_INTERVALS=bedfile,
OUT_STDOUT=AtomicCmd.PIPE,
CHECK_VERSION=BCFTOOLS_VERSION,
)
# Ignore read-groups for pileup
mpileup.add_option("--ignore-RG")
# Reference sequence (FASTA)
mpileup.add_option("--fasta-ref", reference)
# Output compressed VCF
mpileup.add_option("--output-type", "u")
if bedfile:
mpileup.set_option("--regions-file", "%(IN_INTERVALS)s")
apply_options(mpileup, mpileup_options)
genotype = AtomicCmdBuilder(
("bcftools", "call", "-"),
IN_STDIN=mpileup,
IN_BAMFILE=infile,
OUT_STDOUT=outfile,
CHECK_VERSION=BCFTOOLS_VERSION,
)
genotype.set_option("--output-type", "z")
apply_options(genotype, bcftools_options)
CommandNode.__init__(
self,
description="<GenotypeRegions: '%s' -> '%s'>" % (
infile,
outfile,
),
command=ParallelCmds([mpileup.finalize(),
genotype.finalize()]),
dependencies=dependencies,
)
def __init__(self, parameters):
command = ParallelCmds([parameters.commands[key].finalize()
for key in parameters.order])
input_file = parameters.input_file_fq
description = _get_node_description(name="BWA Samse",
input_files_1=input_file,
prefix=parameters.prefix)
CommandNode.__init__(self,
command=command,
description=description,
dependencies=parameters.dependencies)
def __init__(
self,
input_file_fq_1,
input_file_fq_2,
input_file_sai_1,
input_file_sai_2,
output_file,
reference,
prefix,
mapping_options={},
cleanup_options={},
dependencies=(),
):
sampe = _new_bwa_command(
(
"bwa",
"sampe",
prefix,
"%(IN_SAI_1)s",
"%(IN_SAI_2)s",
"%(IN_FQ_1)s",
"%(IN_FQ_2)s",
),
prefix,
IN_SAI_1=input_file_sai_1,
IN_SAI_2=input_file_sai_2,
IN_FQ_1=input_file_fq_1,
IN_FQ_2=input_file_fq_2,
OUT_STDOUT=AtomicCmd.PIPE,
)
cleanup = _new_cleanup_command(sampe,
output_file,
reference,
paired_end=True)
apply_options(sampe, mapping_options)
apply_options(cleanup, cleanup_options)
CommandNode.__init__(
self,
command=ParallelCmds([sampe.finalize(),
cleanup.finalize()]),
description=_get_node_description(
name="BWA Sampe",
input_files_1=input_file_fq_1,
input_files_2=input_file_fq_2,
prefix=prefix,
),
dependencies=dependencies,
)
def __init__(self, parameters):
command = parameters.command.finalize()
self._multi_file_input = len(parameters.input_files) > 1
if self._multi_file_input:
cat = _build_cat_command(parameters.input_files, "uncompressed_input")
command = ParallelCmds((command, cat))
CommandNode.__init__(self,
command=command,
threads=parameters.threads,
description="<AdapterRM (SE): %s -> '%s.*'>"
% (fileutils.describe_files(parameters.input_files),
parameters.output_prefix),
dependencies=parameters.dependencies)
def __init__(self, input_files, destination, dependencies=()):
cat_cmd = factory.new("cat")
cat_cmd.add_multiple_values(input_files)
cat_cmd.set_kwargs(OUT_STDOUT=AtomicCmd.PIPE)
cat_cmd = cat_cmd.finalize()
zip_cmd = AtomicCmd("gzip", IN_STDIN=cat_cmd, OUT_STDOUT=destination)
description = "<Cat %s -> %s>" \
% (fileutils.describe_files(input_files), destination)
CommandNode.__init__(self,
description=description,
command=ParallelCmds((cat_cmd, zip_cmd)),
dependencies=dependencies)
def __init__(self, parameters):
command = ParallelCmds([parameters.commands[key].finalize()
for key in parameters.order])
description \
= _get_node_description(name="BWA",
algorithm='Backtrack',
input_files_1=parameters.input_file,
prefix=parameters.prefix,
threads=parameters.threads)
CommandNode.__init__(self,
command=command,
description=description,
threads=parameters.threads,
dependencies=parameters.dependencies)
def __init__(self, parameters):
_check_bwa_prefix(parameters.prefix)
algorithm = parameters.algorithm.upper()
algorithm += "_PE" if parameters.input_file_2 else "_SE"
desc = _get_node_description(name="BWA",
algorithm=algorithm,
input_files_1=parameters.input_file_1,
input_files_2=parameters.input_file_2,
prefix=parameters.prefix)
command = ParallelCmds([cmd.finalize()
for cmd in parameters.commands.itervalues()])
CommandNode.__init__(self,
command=command,
description=desc,
threads=parameters.threads,
dependencies=parameters.dependencies)
def __init__(self, parameters):
command = ParallelCmds(
[parameters.commands[key].finalize() for key in parameters.order])
algorithm = "PE" if parameters.input_file_2 else "SE"
description \
= _get_node_description(name="Bowtie2",
algorithm=algorithm,
input_files_1=parameters.input_file_1,
input_files_2=parameters.input_file_2,
prefix=parameters.prefix,
threads=parameters.threads)
CommandNode.__init__(self,
command=command,
description=description,
threads=parameters.threads,
dependencies=parameters.dependencies)
def __init__(
self,
reference,
input_files,
output_file,
directory,
options={},
dependencies=(),
):
stats_out_fname = "Stats_out_MCMC_correct_prob.csv"
merge = merge_bam_files_command(input_files)
command = AtomicCmdBuilder(
[
"mapDamage",
"--rescale-only",
"-i",
"-",
"-d",
"%(TEMP_DIR)s",
"-r",
"%(IN_REFERENCE)s",
"--rescale-out",
"%(OUT_BAM)s",
],
IN_STDIN=merge,
IN_REFERENCE=reference,
TEMP_OUT_LOG="Runtime_log.txt",
TEMP_OUT_CSV=stats_out_fname,
OUT_BAM=output_file,
CHECK_VERSION=MAPDAMAGE_VERSION,
)
apply_options(command, options)
self._directory = directory
CommandNode.__init__(
self,
command=ParallelCmds([merge, command.finalize()]),
description="<mapDamage (rescale): %s -> %r>"
% (describe_files(merge.input_files), output_file,),
dependencies=dependencies,
)
def __init__(self,
config,
input_bams,
output_bam,
keep_dupes=True,
dependencies=()):
merge = merge_bam_files_command(input_bams)
builder = factory.new("rmdup_collapsed")
builder.set_kwargs(IN_STDIN=merge, OUT_STDOUT=output_bam)
if not keep_dupes:
builder.set_option("--remove-duplicates")
description = "<FilterCollapsedBAM: %s>" % (describe_files(
merge.input_files), )
CommandNode.__init__(
self,
command=ParallelCmds([merge, builder.finalize()]),
description=description,
dependencies=dependencies,
)
def __init__(self,
config,
input_bams,
command,
index_format=None,
description=None,
threads=1,
dependencies=()):
self._input_bams = safe_coerce_to_tuple(input_bams)
self._index_format = index_format
if not self._input_bams:
raise ValueError("No input BAM files specified!")
elif len(self._input_bams) > 1 and index_format:
raise ValueError("BAM index cannot be required for > 1 file")
elif index_format not in (None, ".bai", ".csi"):
raise ValueError("Unknown index format %r" % (index_format, ))
if len(self._input_bams) > 1:
merge = picard_command(config, "MergeSamFiles")
merge.set_option("SO", "coordinate", sep="=")
merge.set_option("COMPRESSION_LEVEL", 0, sep="=")
merge.set_option("OUTPUT", "%(TEMP_OUT_BAM)s", sep="=")
# Validation is mostly left to manual ValidateSamFile runs; this
# is because .csi indexed BAM records can have "invalid" bins.
merge.set_option("VALIDATION_STRINGENCY", "LENIENT", sep="=")
merge.add_multiple_options("I", input_bams, sep="=")
merge.set_kwargs(TEMP_OUT_BAM=self.PIPE_FILE)
command = ParallelCmds([merge.finalize(), command])
CommandNode.__init__(self,
command=command,
description=description,
threads=threads,
dependencies=dependencies)
def __init__(
self,
input_file_fq,
input_file_sai,
output_file,
reference,
prefix,
mapping_options={},
cleanup_options={},
dependencies=(),
):
samse = _new_bwa_command(
("bwa", "samse"),
prefix,
IN_FILE_SAI=input_file_sai,
IN_FILE_FQ=input_file_fq,
OUT_STDOUT=AtomicCmd.PIPE,
)
samse.add_value(prefix)
samse.add_value("%(IN_FILE_SAI)s")
samse.add_value("%(IN_FILE_FQ)s")
cleanup = _new_cleanup_command(samse, output_file, reference)
apply_options(samse, mapping_options)
apply_options(cleanup, cleanup_options)
CommandNode.__init__(
self,
command=ParallelCmds([samse.finalize(),
cleanup.finalize()]),
description=_get_node_description(name="BWA Samse",
input_files_1=input_file_fq,
prefix=prefix),
dependencies=dependencies,
)
def test_parallel_commands__reject_empty_commandset():
with pytest.raises(CmdError):
ParallelCmds([])
def test_parallel_commands__join_after_run(temp_folder):
cmds = ParallelCmds([AtomicCmd("true") for _ in range(3)])
cmds.run(temp_folder)
assert_equal(cmds.join(), [0, 0, 0])
def test_parallel_commands__join_failure_3(temp_folder):
mocks = _setup_mocks_for_failure(True, True, False)
cmds = ParallelCmds(mocks)
cmds.run(temp_folder)
assert_equal(cmds.join(), ['SIGTERM', 'SIGTERM', 1])
def test_parallel_commands__reject_sequential():
command = AtomicCmd(["ls"])
seqcmd = SequentialCmds([command])
with pytest.raises(CmdError):
ParallelCmds([seqcmd])
def test_parallel_commands__reject_noncommand():
with pytest.raises(CmdError):
ParallelCmds([object()])
def __init__(
self,
input_file_1,
input_file_2,
output_file,
reference,
prefix,
threads=2,
log_file=None,
mapping_options={},
cleanup_options={},
dependencies=(),
):
# Setting IN_FILE_2 to None makes AtomicCmd ignore this key
aln = _bowtie2_template(
("bowtie2", ),
prefix,
OUT_STDOUT=AtomicCmd.PIPE,
CHECK_VERSION=BOWTIE2_VERSION,
)
aln.set_option("-x", prefix)
if log_file is not None:
aln.set_kwargs(OUT_STDERR=log_file)
if input_file_1 and not input_file_2:
aln.add_option("-U", input_file_1)
elif input_file_1 and input_file_2:
aln.add_option("-1", input_file_1)
aln.add_option("-2", input_file_2)
else:
raise NodeError("Input 1, OR both input 1 and input 2 must "
"be specified for Bowtie2 node")
max_threads = _get_max_threads(reference, threads)
aln.set_option("--threads", max_threads)
cleanup = _new_cleanup_command(aln,
output_file,
reference,
paired_end=input_file_1
and input_file_2)
apply_options(aln, mapping_options)
apply_options(cleanup, cleanup_options)
algorithm = "PE" if input_file_2 else "SE"
description = _get_node_description(
name="Bowtie2",
algorithm=algorithm,
input_files_1=input_file_1,
input_files_2=input_file_2,
prefix=prefix,
threads=threads,
)
CommandNode.__init__(
self,
command=ParallelCmds([aln.finalize(),
cleanup.finalize()]),
description=description,
threads=threads,
dependencies=dependencies,
)
def test_sequential_commands__accept_parallel():
command = AtomicCmd(["ls"])
parcmd = ParallelCmds([command])
SequentialCmds([parcmd])
def _do_test_parallel_commands__ready_single(value):
cmd_mock = flexmock(AtomicCmd(["ls"]))
cmd_mock.should_receive('ready').and_return(value).at_least.once
cmds = ParallelCmds([cmd_mock])
assert_equal(cmds.ready(), value)
