def tobam_cl(data, out_file, is_paired=False):
"""Prepare command line for producing de-duplicated sorted output.
- If no deduplication, sort and prepare a BAM file.
- If paired, then use samblaster and prepare discordant outputs.
- If unpaired, use biobambam's bammarkduplicates
"""
do_dedup = _check_dedup(data)
umi_consensus = dd.get_umi_consensus(data)
with file_transaction(data, out_file) as tx_out_file:
if not do_dedup:
yield (sam_to_sortbam_cl(data, tx_out_file), tx_out_file)
elif umi_consensus:
yield (_sam_to_grouped_umi_cl(data, umi_consensus,
tx_out_file), tx_out_file)
elif is_paired and _need_sr_disc_reads(data) and not _too_many_contigs(
dd.get_ref_file(data)):
sr_file = "%s-sr.bam" % os.path.splitext(out_file)[0]
disc_file = "%s-disc.bam" % os.path.splitext(out_file)[0]
with file_transaction(data, sr_file) as tx_sr_file:
with file_transaction(data, disc_file) as tx_disc_file:
yield (samblaster_dedup_sort(data, tx_out_file, tx_sr_file,
tx_disc_file), tx_out_file)
else:
yield (_biobambam_dedup_sort(data, tx_out_file), tx_out_file)
def _set_align_split_size(data):
"""Set useful align_split_size, generating an estimate if it doesn't exist.
We try to split on larger inputs and avoid too many pieces, aiming for size
chunks of 5Gb or at most 50 maximum splits.
The size estimate used in calculations is 20 million reads for ~5Gb.
For UMI calculations we skip splitting since we're going to align and
re-align after consensus.
"""
target_size = 5 # Gb
target_size_reads = 20 # million reads
max_splits = 100 # Avoid too many pieces, causing merge memory problems
val = tz.get_in(["config", "algorithm", "align_split_size"], data)
umi_consensus = dd.get_umi_consensus(data)
if val is None:
if not umi_consensus:
total_size = 0 # Gb
for fname in data.get("files", []):
if os.path.exists(fname):
total_size += os.path.getsize(fname) / (1024.0 * 1024.0 *
1024.0)
# Only set if we have files and are bigger than the target size
if total_size > target_size:
data["config"]["algorithm"]["align_split_size"] = \
int(1e6 * _pick_align_split_size(total_size, target_size,
target_size_reads, max_splits))
elif val:
assert not umi_consensus, "Cannot set align_split_size to %s with UMI conensus specified" % val
return data
def get_qc_tools(data):
"""Retrieve a list of QC tools to use based on configuration and analysis type.
Uses defaults if previously set.
"""
if dd.get_algorithm_qc(data):
return dd.get_algorithm_qc(data)
analysis = data["analysis"].lower()
to_run = []
if "fastqc" not in dd.get_tools_off(data):
to_run.append("fastqc")
if any([tool in dd.get_tools_on(data)
for tool in ["qualimap", "qualimap_full"]]):
to_run.append("qualimap")
if analysis.startswith("rna-seq"):
if "qualimap" not in dd.get_tools_off(data):
if gtf.is_qualimap_compatible(dd.get_gtf_file(data)):
to_run.append("qualimap_rnaseq")
else:
logger.debug("GTF not compatible with Qualimap, skipping.")
if analysis.startswith("smallrna-seq"):
to_run.append("small-rna")
if not analysis.startswith("smallrna-seq"):
to_run.append("samtools")
if tz.get_in(["config", "algorithm", "kraken"], data):
to_run.append("kraken")
if analysis.startswith(("standard", "variant", "variant2")):
to_run += ["qsignature", "coverage", "variants", "picard"]
if vcfutils.get_paired([data]):
to_run += ["viral"]
if damage.should_filter([data]):
to_run += ["damage"]
if dd.get_umi_consensus(data):
to_run += ["umi"]
return to_run
def _set_align_split_size(data):
"""Set useful align_split_size, generating an estimate if it doesn't exist.
We try to split on larger inputs and avoid too many pieces, aiming for size
chunks of 5Gb or at most 50 maximum splits.
The size estimate used in calculations is 20 million reads for ~5Gb.
For UMI calculations we skip splitting since we're going to align and
re-align after consensus.
"""
target_size = 5 # Gb
target_size_reads = 20 # million reads
max_splits = 100 # Avoid too many pieces, causing merge memory problems
val = tz.get_in(["config", "algorithm", "align_split_size"], data)
umi_consensus = dd.get_umi_consensus(data)
if val is None:
if not umi_consensus:
total_size = 0 # Gb
# Use original files if we might have reduced the size of our prepped files
input_files = data.get("files_orig", []) if dd.get_save_diskspace(data) else data.get("files", [])
for fname in input_files:
if os.path.exists(fname):
total_size += os.path.getsize(fname) / (1024.0 * 1024.0 * 1024.0)
# Only set if we have files and are bigger than the target size
if total_size > target_size:
data["config"]["algorithm"]["align_split_size"] = \
int(1e6 * _pick_align_split_size(total_size, target_size,
target_size_reads, max_splits))
elif val:
assert not umi_consensus, "Cannot set align_split_size to %s with UMI conensus specified" % val
return data
def get_qc_tools(data):
"""Retrieve a list of QC tools to use based on configuration and analysis type.
Uses defaults if previously set.
"""
if dd.get_algorithm_qc(data):
return dd.get_algorithm_qc(data)
analysis = data["analysis"].lower()
to_run = []
if "fastqc" not in dd.get_tools_off(data):
to_run.append("fastqc")
if any([
tool in dd.get_tools_on(data)
for tool in ["qualimap", "qualimap_full"]
]):
to_run.append("qualimap")
if analysis.startswith("rna-seq"):
if gtf.is_qualimap_compatible(dd.get_gtf_file(data)):
to_run.append("qualimap_rnaseq")
else:
logger.debug("GTF not compatible with Qualimap, skipping.")
if analysis.startswith("smallrna-seq"):
to_run.append("small-rna")
if not analysis.startswith("smallrna-seq"):
to_run.append("samtools")
to_run.append("gemini")
if tz.get_in(["config", "algorithm", "kraken"], data):
to_run.append("kraken")
if analysis.startswith(("standard", "variant", "variant2")):
to_run += ["qsignature", "coverage", "variants", "picard"]
if dd.get_umi_consensus(data):
to_run += ["umi"]
return to_run
def get_qc_tools(data):
"""Retrieve a list of QC tools to use based on configuration and analysis type.
Uses defaults if previously set.
"""
if dd.get_algorithm_qc(data):
return dd.get_algorithm_qc(data)
analysis = data["analysis"].lower()
to_run = []
if tz.get_in(["config", "algorithm", "kraken"], data):
to_run.append("kraken")
if "fastqc" not in dd.get_tools_off(data):
to_run.append("fastqc")
if any([
tool in dd.get_tools_on(data)
for tool in ["qualimap", "qualimap_full"]
]):
to_run.append("qualimap")
if analysis.startswith("rna-seq") or analysis == "smallrna-seq":
if "qualimap" not in dd.get_tools_off(data):
if gtf.is_qualimap_compatible(dd.get_gtf_file(data)):
to_run.append("qualimap_rnaseq")
else:
logger.debug("GTF not compatible with Qualimap, skipping.")
if analysis.startswith("chip-seq"):
to_run.append("chipqc")
if dd.get_chip_method(data) == "atac":
to_run.append("ataqv")
if analysis.startswith("smallrna-seq"):
to_run.append("small-rna")
to_run.append("atropos")
if "coverage_qc" not in dd.get_tools_off(data):
to_run.append("samtools")
if dd.has_variantcalls(data):
if "coverage_qc" not in dd.get_tools_off(data):
to_run += ["coverage", "picard"]
to_run += ["qsignature", "variants"]
if vcfanno.is_human(data):
to_run += ["peddy"]
if "contamination" not in dd.get_tools_off(data):
to_run += ["contamination"]
if vcfutils.get_paired_phenotype(data):
if "viral" not in dd.get_tools_off(data):
to_run += ["viral"]
if damage.should_filter([data]):
to_run += ["damage"]
if dd.get_umi_consensus(data):
to_run += ["umi"]
if tz.get_in(["config", "algorithm", "preseq"], data):
to_run.append("preseq")
to_run = [tool for tool in to_run if tool not in dd.get_tools_off(data)]
to_run.sort()
return to_run
def _variant_checkpoints(samples):
"""Check sample configuration to identify required steps in analysis.
"""
checkpoints = {}
checkpoints["vc"] = any([dd.get_variantcaller(d) or d.get("vrn_file") for d in samples])
checkpoints["sv"] = any([dd.get_svcaller(d) for d in samples])
checkpoints["jointvc"] = any([(dd.get_jointcaller(d) or ("gvcf" in dd.get_tools_on(d))) and dd.get_batch(d)
for d in samples])
checkpoints["hla"] = any([dd.get_hlacaller(d) for d in samples])
checkpoints["align"] = any([(dd.get_aligner(d) or dd.get_bam_clean(d)) for d in samples])
checkpoints["align_split"] = not all([(dd.get_align_split_size(d) is False or
not dd.get_aligner(d))
for d in samples])
checkpoints["umi"] = any([dd.get_umi_consensus(d) for d in samples])
return checkpoints
def _variant_checkpoints(samples):
"""Check sample configuration to identify required steps in analysis.
"""
checkpoints = {}
checkpoints["vc"] = any([dd.get_variantcaller(d) or d.get("vrn_file") for d in samples])
checkpoints["sv"] = any([dd.get_svcaller(d) for d in samples])
checkpoints["jointvc"] = any([(dd.get_jointcaller(d) or "gvcf" in dd.get_tools_on(d))
for d in samples])
checkpoints["hla"] = any([dd.get_hlacaller(d) for d in samples])
checkpoints["align"] = any([(dd.get_aligner(d) or dd.get_bam_clean(d)) for d in samples])
checkpoints["align_split"] = not all([(dd.get_align_split_size(d) is False or
not dd.get_aligner(d))
for d in samples])
checkpoints["archive"] = any([dd.get_archive(d) for d in samples])
checkpoints["umi"] = any([dd.get_umi_consensus(d) for d in samples])
checkpoints["ensemble"] = any([dd.get_ensemble(d) for d in samples])
checkpoints["cancer"] = any(dd.get_phenotype(d) in ["tumor"] for d in samples)
return checkpoints
def _set_align_split_size(data):
"""Set useful align_split_size, generating an estimate if it doesn't exist.
We try to split on larger inputs and avoid too many pieces, aiming for size
chunks of 5Gb or at most 50 maximum splits.
The size estimate used in calculations is 20 million reads for ~5Gb.
For UMI calculations we skip splitting since we're going to align and
re-align after consensus.
For CWL runs, we pick larger split sizes to avoid overhead of staging each chunk.
"""
if cwlutils.is_cwl_run(data):
target_size = 20 # Gb
target_size_reads = 80 # million reads
else:
target_size = 5 # Gb
target_size_reads = 20 # million reads
max_splits = 100 # Avoid too many pieces, causing merge memory problems
val = dd.get_align_split_size(data)
umi_consensus = dd.get_umi_consensus(data)
if val is None:
if not umi_consensus:
total_size = 0 # Gb
# Use original files if we might have reduced the size of our prepped files
input_files = data.get(
"files_orig", []) if dd.get_save_diskspace(data) else data.get(
"files", [])
for fname in input_files:
if os.path.exists(fname):
total_size += os.path.getsize(fname) / (1024.0 * 1024.0 *
1024.0)
# Only set if we have files and are bigger than the target size
if total_size > target_size:
data["config"]["algorithm"]["align_split_size"] = \
int(1e6 * _pick_align_split_size(total_size, target_size,
target_size_reads, max_splits))
elif val:
assert not umi_consensus, "Cannot set align_split_size to %s with UMI conensus specified" % val
return data
def tobam_cl(data, out_file, is_paired=False):
"""Prepare command line for producing de-duplicated sorted output.
- If no deduplication, sort and prepare a BAM file.
- If paired, then use samblaster and prepare discordant outputs.
- If unpaired, use biobambam's bammarkduplicates
"""
do_dedup = _check_dedup(data)
umi_consensus = dd.get_umi_consensus(data)
with file_transaction(data, out_file) as tx_out_file:
if not do_dedup:
yield (sam_to_sortbam_cl(data, tx_out_file), tx_out_file)
elif umi_consensus:
yield (_sam_to_grouped_umi_cl(data, umi_consensus, tx_out_file), tx_out_file)
elif is_paired and _need_sr_disc_reads(data) and not _too_many_contigs(dd.get_ref_file(data)):
sr_file = "%s-sr.bam" % os.path.splitext(out_file)[0]
disc_file = "%s-disc.bam" % os.path.splitext(out_file)[0]
with file_transaction(data, sr_file) as tx_sr_file:
with file_transaction(data, disc_file) as tx_disc_file:
yield (samblaster_dedup_sort(data, tx_out_file, tx_sr_file, tx_disc_file), tx_out_file)
else:
yield (_biobambam_dedup_sort(data, tx_out_file), tx_out_file)
def process_alignment(data, alt_input=None):
"""Do an alignment of fastq files, preparing a sorted BAM output file.
"""
data = cwlutils.normalize_missing(utils.to_single_data(data))
data = cwlutils.unpack_tarballs(data, data)
fastq1, fastq2 = dd.get_input_sequence_files(data)
if alt_input:
fastq1, fastq2 = alt_input
config = data["config"]
aligner = config["algorithm"].get("aligner", None)
if fastq1 and objectstore.file_exists_or_remote(fastq1) and aligner:
logger.info("Aligning lane %s with %s aligner" %
(data["rgnames"]["lane"], aligner))
data = align_to_sort_bam(fastq1, fastq2, aligner, data)
if dd.get_correct_umis(data):
data["work_bam"] = postalign.correct_umis(data)
if dd.get_umi_consensus(data):
data["umi_bam"] = dd.get_work_bam(data)
if fastq2:
f1, f2, avg_cov = postalign.umi_consensus(data)
data["config"]["algorithm"]["rawumi_avg_cov"] = avg_cov
del data["config"]["algorithm"]["umi_type"]
data["config"]["algorithm"]["mark_duplicates"] = False
data = align_to_sort_bam(f1, f2, aligner, data)
else:
raise ValueError(
"Single fastq input for UMI processing; fgbio needs paired reads: %s"
% dd.get_sample_name(data))
data = _add_supplemental_bams(data)
elif fastq1 and objectstore.file_exists_or_remote(
fastq1) and fastq1.endswith(".bam"):
sort_method = config["algorithm"].get("bam_sort")
bamclean = config["algorithm"].get("bam_clean")
if bamclean is True or bamclean == "picard":
if sort_method and sort_method != "coordinate":
raise ValueError(
"Cannot specify `bam_clean: picard` with `bam_sort` other than coordinate: %s"
% sort_method)
ref_file = dd.get_ref_file(data)
out_bam = cleanbam.picard_prep(fastq1, data["rgnames"], ref_file,
data["dirs"], data)
elif bamclean == "fixrg":
out_bam = cleanbam.fixrg(fastq1, data["rgnames"],
dd.get_ref_file(data), data["dirs"], data)
elif bamclean == "remove_extracontigs":
out_bam = cleanbam.remove_extracontigs(fastq1, data)
elif sort_method:
runner = broad.runner_from_path("picard", config)
out_file = os.path.join(
data["dirs"]["work"], "{}-sort.bam".format(
os.path.splitext(os.path.basename(fastq1))[0]))
if not utils.file_exists(out_file):
work_dir = utils.safe_makedir(
os.path.join(dd.get_work_dir(data), "bamclean",
dd.get_sample_name(data)))
out_file = os.path.join(
work_dir, "{}-sort.bam".format(dd.get_sample_name(data)))
out_bam = runner.run_fn("picard_sort", fastq1, sort_method,
out_file)
else:
out_bam = _link_bam_file(
fastq1,
os.path.join(dd.get_work_dir(data), "prealign",
dd.get_sample_name(data)), data)
bam.index(out_bam, data["config"])
bam.check_header(out_bam, data["rgnames"], dd.get_ref_file(data),
data["config"])
dedup_bam = postalign.dedup_bam(out_bam, data)
bam.index(dedup_bam, data["config"])
data["work_bam"] = dedup_bam
elif fastq1 and objectstore.file_exists_or_remote(
fastq1) and fastq1.endswith(".cram"):
data["work_bam"] = fastq1
elif fastq1 is None and not dd.get_aligner(data):
data["config"]["algorithm"]["variantcaller"] = False
data["work_bam"] = None
elif not fastq1:
raise ValueError("No 'files' specified for input sample: %s" %
dd.get_sample_name(data))
elif "kraken" in config["algorithm"]: # kraken doesn's need bam
pass
else:
raise ValueError(
"Could not process input file from sample configuration. \n" +
fastq1 + "\nIs the path to the file correct or is empty?\n" +
"If it is a fastq file (not pre-aligned BAM or CRAM), "
"is an aligner specified in the input configuration?")
if data.get("work_bam"):
# Add stable 'align_bam' target to use for retrieving raw alignment
data["align_bam"] = data["work_bam"]
data = _add_hla_files(data)
return [[data]]
def process_alignment(data, alt_input=None):
"""Do an alignment of fastq files, preparing a sorted BAM output file.
"""
data = cwlutils.normalize_missing(utils.to_single_data(data))
data = cwlutils.unpack_tarballs(data, data)
fastq1, fastq2 = dd.get_input_sequence_files(data)
if alt_input:
fastq1, fastq2 = alt_input
config = data["config"]
aligner = config["algorithm"].get("aligner", None)
if fastq1 and objectstore.file_exists_or_remote(fastq1) and aligner:
logger.info("Aligning lane %s with %s aligner" % (data["rgnames"]["lane"], aligner))
data = align_to_sort_bam(fastq1, fastq2, aligner, data)
if dd.get_umi_consensus(data):
data["umi_bam"] = dd.get_work_bam(data)
if fastq2:
f1, f2 = postalign.umi_consensus(data)
del data["config"]["algorithm"]["umi_type"]
data["config"]["algorithm"]["mark_duplicates"] = False
data = align_to_sort_bam(f1, f2, aligner, data)
data = _add_supplemental_bams(data)
elif fastq1 and objectstore.file_exists_or_remote(fastq1) and fastq1.endswith(".bam"):
sort_method = config["algorithm"].get("bam_sort")
bamclean = config["algorithm"].get("bam_clean")
if bamclean is True or bamclean == "picard":
if sort_method and sort_method != "coordinate":
raise ValueError("Cannot specify `bam_clean: picard` with `bam_sort` other than coordinate: %s"
% sort_method)
out_bam = cleanbam.picard_prep(fastq1, data["rgnames"], dd.get_ref_file(data), data["dirs"],
data)
elif bamclean == "fixrg":
out_bam = cleanbam.fixrg(fastq1, data["rgnames"], dd.get_ref_file(data), data["dirs"], data)
elif bamclean == "remove_extracontigs":
out_bam = cleanbam.remove_extracontigs(fastq1, data)
data["reference"]["fasta"] = bam.ref_file_from_bam(out_bam, data)
elif sort_method:
runner = broad.runner_from_path("picard", config)
out_file = os.path.join(data["dirs"]["work"], "{}-sort.bam".format(
os.path.splitext(os.path.basename(fastq1))[0]))
out_bam = runner.run_fn("picard_sort", fastq1, sort_method, out_file)
else:
out_bam = link_bam_file(fastq1, os.path.join(data["dirs"]["work"], "prealign",
data["rgnames"]["sample"]))
bam.index(out_bam, data["config"])
bam.check_header(out_bam, data["rgnames"], dd.get_ref_file(data), data["config"])
dedup_bam = postalign.dedup_bam(out_bam, data)
bam.index(dedup_bam, data["config"])
data["work_bam"] = dedup_bam
elif fastq1 and objectstore.file_exists_or_remote(fastq1) and fastq1.endswith(".cram"):
data["work_bam"] = fastq1
elif fastq1 is None and "vrn_file" in data:
data["config"]["algorithm"]["variantcaller"] = False
data["work_bam"] = None
elif not fastq1:
raise ValueError("No 'files' specified for input sample: %s" % dd.get_sample_name(data))
elif "kraken" in config["algorithm"]: # kraken doesn's need bam
pass
else:
raise ValueError("Could not process input file from sample configuration. \n" +
fastq1 +
"\nIs the path to the file correct or is empty?\n" +
"If it is a fastq file (not pre-aligned BAM or CRAM), "
"is an aligner specified in the input configuration?")
if data.get("work_bam"):
# Add stable 'align_bam' target to use for retrieving raw alignment
data["align_bam"] = data["work_bam"]
data = _add_hla_files(data)
return [[data]]