def summary(*samples):
"""Summarize all quality metrics together"""
samples = list(utils.flatten(samples))
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
out_dir = utils.safe_makedir(os.path.join(work_dir, "qc", "multiqc"))
out_data = os.path.join(out_dir, "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
file_list = os.path.join(out_dir, "list_files.txt")
work_samples = [cwlutils.unpack_tarballs(utils.deepish_copy(x), x) for x in samples]
work_samples = _report_summary(work_samples, os.path.join(out_dir, "report"))
if not utils.file_exists(out_file):
with tx_tmpdir(samples[0], work_dir) as tx_out:
in_files = _get_input_files(work_samples, out_dir, tx_out)
in_files += _merge_metrics(work_samples, out_dir)
if _one_exists(in_files):
with utils.chdir(out_dir):
_create_config_file(out_dir, work_samples)
input_list_file = _create_list_file(in_files, file_list)
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
else:
export_tmp = ""
path_export = utils.local_path_export()
other_opts = config_utils.get_resources("multiqc", samples[0]["config"]).get("options", [])
other_opts = " ".join([str(x) for x in other_opts])
cmd = "{path_export}{export_tmp} {multiqc} -f -l {input_list_file} {other_opts} -o {tx_out}"
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
samples = _group_by_sample_and_batch(samples)
if utils.file_exists(out_file) and samples:
data_files = set()
for i, data in enumerate(samples):
data_files.add(os.path.join(out_dir, "report", "metrics", dd.get_sample_name(data) + "_bcbio.txt"))
data_files.add(os.path.join(out_dir, "report", "metrics", "target_info.yaml"))
data_files.add(os.path.join(out_dir, "multiqc_config.yaml"))
data_files = [f for f in data_files if f and utils.file_exists(f)]
if "summary" not in samples[0]:
samples[0]["summary"] = {}
samples[0]["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
data_json = os.path.join(out_dir, "multiqc_data", "multiqc_data.json")
data_json_final = _save_uploaded_data_json(samples, data_json, os.path.join(out_dir, "multiqc_data"))
if data_json_final:
samples[0]["summary"]["multiqc"]["secondary"].append(data_json_final)
file_list_final = _save_uploaded_file_list(samples, file_list, out_dir)
if file_list_final:
samples[0]["summary"]["multiqc"]["secondary"].append(file_list_final)
return [[data] for data in samples]
def summary(*samples):
"""Summarize all quality metrics together"""
samples = list(utils.flatten(samples))
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
out_dir = utils.safe_makedir(os.path.join(work_dir, "qc", "multiqc"))
out_data = os.path.join(out_dir, "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
file_list = os.path.join(out_dir, "list_files.txt")
work_samples = [cwlutils.unpack_tarballs(utils.deepish_copy(x), x) for x in samples]
work_samples = _report_summary(work_samples, os.path.join(out_dir, "report"))
if not utils.file_exists(out_file):
with tx_tmpdir(samples[0], work_dir) as tx_out:
in_files = _get_input_files(work_samples, out_dir, tx_out)
in_files += _merge_metrics(work_samples, out_dir)
if _one_exists(in_files):
with utils.chdir(out_dir):
_create_config_file(out_dir, work_samples)
input_list_file = _create_list_file(in_files, file_list)
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
else:
export_tmp = ""
path_export = utils.local_path_export()
other_opts = config_utils.get_resources("multiqc", samples[0]["config"]).get("options", [])
other_opts = " ".join([str(x) for x in other_opts])
cmd = "{path_export}{export_tmp} {multiqc} -f -l {input_list_file} {other_opts} -o {tx_out}"
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
out = []
for i, data in enumerate(_group_by_samplename(samples)):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.yaml"))
data_files += glob.glob(os.path.join(out_dir, "report", "*.R*"))
data_files += glob.glob(os.path.join(out_dir, "multiqc_config.yaml"))
data_files.append(file_list)
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
file_list_final = _save_uploaded_file_list(samples, file_list, out_dir)
if file_list_final:
data["summary"]["multiqc"]["secondary"].append(file_list_final)
out.append([data])
return out
def summary(*samples):
"""Summarize all quality metrics together"""
samples = utils.unpack_worlds(samples)
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
file_fapths = []
opts = ""
out_dir = os.path.join(work_dir, "multiqc")
out_data = os.path.join(work_dir, "multiqc", "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
samples = _report_summary(samples, os.path.join(out_dir, "report"))
for data in samples:
for program, pfiles in tz.get_in(["summary", "qc"], data, {}).iteritems():
if isinstance(pfiles, dict):
pfiles = [pfiles["base"]] + pfiles["secondary"]
elif isinstance(pfiles, basestring):
pfiles = [pfiles]
file_fapths.extend(pfiles)
file_fapths.append(os.path.join(out_dir, "report", "metrics", "target_info.yaml"))
# XXX temporary workaround until we can handle larger inputs through MultiQC
file_fapths = list(set(file_fapths))
# Back compatible -- to migrate to explicit specifications in input YAML
file_fapths += ["trimmed", "htseq-count/*summary"]
if not utils.file_exists(out_file):
with utils.chdir(work_dir):
file_fapths = [fpath for fpath in file_fapths if _check_multiqc_input(fpath) and _is_good_file_for_multiqc(fpath)]
input_list_file = _create_list_file(file_fapths)
export_tmp = ""
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
if input_list_file:
cmd = "{export_tmp} {multiqc} -f -l {input_list_file} -o {tx_out} {opts}"
with tx_tmpdir(data, work_dir) as tx_out:
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
out = []
for i, data in enumerate(samples):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report", "*.R*"))
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
out.append(data)
return [[fpath] for fpath in out]
def summary(*samples):
"""Summarize all quality metrics together"""
samples = utils.unpack_worlds(samples)
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
folders = []
opts = ""
out_dir = os.path.join(work_dir, "multiqc")
out_data = os.path.join(work_dir, "multiqc", "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
samples = _report_summary(samples, os.path.join(out_dir, "report"))
for data in samples:
for program, pfiles in tz.get_in(["summary", "qc"], data, {}).iteritems():
if isinstance(pfiles, dict):
pfiles = pfiles["base"]
folders.append(os.path.dirname(pfiles))
# XXX temporary workaround until we can handle larger inputs through MultiQC
folders = list(set(folders))
if len(folders) > 250:
logger.warning("Too many samples for MultiQC, only using first 250 entries.")
folders = folders[:250]
opts = "--flat"
# Back compatible -- to migrate to explicit specifications in input YAML
folders += ["trimmed", "htseq-count/*summary"]
if not utils.file_exists(out_file):
with utils.chdir(work_dir):
input_dir = " ".join([_check_multiqc_input(d) for d in folders])
export_tmp = ""
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
if input_dir.strip():
cmd = "{export_tmp} {multiqc} -f {input_dir} -o {tx_out} {opts}"
with tx_tmpdir(data, work_dir) as tx_out:
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
out = []
for i, data in enumerate(samples):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report", "*.R*"))
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
out.append(data)
return [[d] for d in out]
def summary(*samples):
"""Summarize all quality metrics together"""
samples = utils.unpack_worlds(samples)
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
out_dir = utils.safe_makedir(os.path.join(work_dir, "qc", "multiqc"))
out_data = os.path.join(out_dir, "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
file_list = os.path.join(out_dir, "list_files.txt")
samples = _report_summary(samples, os.path.join(out_dir, "report"))
if not utils.file_exists(out_file):
with tx_tmpdir(samples[0], work_dir) as tx_out:
in_files = _get_input_files(samples, out_dir, tx_out)
in_files += _merge_metrics(samples, out_dir)
if _one_exists(in_files):
with utils.chdir(out_dir):
_create_config_file(out_dir, samples)
input_list_file = _create_list_file(in_files, file_list)
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
else:
export_tmp = ""
path_export = utils.local_path_export()
cmd = "{path_export}{export_tmp} {multiqc} -f -l {input_list_file} -o {tx_out}"
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
out = []
for i, data in enumerate(_group_by_samplename(samples)):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.yaml"))
data_files += glob.glob(os.path.join(out_dir, "report", "*.R*"))
data_files.append(file_list)
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
file_list_final = _save_uploaded_file_list(samples, file_list, out_dir)
if file_list_final:
data["summary"]["multiqc"]["secondary"].append(file_list_final)
out.append([data])
return out
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", sailfish.get_build_string(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambiguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
if dd.get_transcriptome_fasta(data):
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data, out_dir)
assert file_exists(gtf_fa), "%s was not found, exiting." % gtf_fa
tmpdir = dd.get_tmp_dir(data)
out_file = os.path.join(out_dir, "versionInfo.json")
if file_exists(out_file):
return out_dir
files = dd.get_input_sequence_files(data)
readlength = bam.fastq.estimate_read_length(files[0])
if readlength % 2 == 0:
readlength -= 1
kmersize = min(readlength, 31)
with file_transaction(data, out_dir) as tx_out_dir:
cmd = "{salmon} index -k {kmersize} -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def rapmap_index(gtf_file, ref_file, algorithm, data, out_dir):
valid_indexes = ["pseudoindex", "quasiindex"]
index_type = algorithm + "index"
assert index_type in valid_indexes, \
"RapMap only supports %s indices." % valid_indexes
out_dir = os.path.join(out_dir, index_type, dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
rapmap = config_utils.get_program("rapmap", dd.get_config(data))
# use user supplied transcriptome FASTA file if it exists
if dd.get_transcriptome_fasta(data):
out_dir = os.path.join(out_dir, index_type, dd.get_genome_build(data))
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data)
tmpdir = dd.get_tmp_dir(data)
if file_exists(out_dir + "rapidx.jfhash"):
return out_dir
files = dd.get_input_sequence_files(data)
kmersize = sailfish.pick_kmersize(files[0])
message = "Creating rapmap {index_type} for {gtf_fa} with {kmersize} bp kmers."
with file_transaction(out_dir) as tx_out_dir:
cmd = "{rapmap} {index_type} -k {kmersize} -i {tx_out_dir} -t {gtf_fa}"
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", sailfish.get_build_string(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambiguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
if dd.get_transcriptome_fasta(data):
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data)
assert file_exists(gtf_fa), "%s was not found, exiting." % gtf_fa
tmpdir = dd.get_tmp_dir(data)
out_file = os.path.join(out_dir, "versionInfo.json")
if file_exists(out_file):
logger.info("Transcriptome index for %s detected, skipping building." %
gtf_fa)
return out_dir
files = dd.get_input_sequence_files(data)
readlength = bam.fastq.estimate_read_length(files[0])
if readlength % 2 == 0:
readlength -= 1
kmersize = min(readlength, 31)
with file_transaction(data, out_dir) as tx_out_dir:
cmd = "{salmon} index -k {kmersize} -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def rapmap_index(gtf_file, ref_file, algorithm, data, out_dir):
valid_indexes = ["pseudoindex", "quasiindex"]
index_type = algorithm + "index"
assert index_type in valid_indexes, \
"RapMap only supports %s indices." % valid_indexes
out_dir = os.path.join(out_dir, index_type, dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
rapmap = config_utils.get_program("rapmap", dd.get_config(data))
# use user supplied transcriptome FASTA file if it exists
if dd.get_transcriptome_fasta(data):
out_dir = os.path.join(out_dir, index_type, dd.get_genome_build(data))
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data)
tmpdir = dd.get_tmp_dir(data)
if file_exists(out_dir + "rapidx.jfhash"):
return out_dir
files = dd.get_input_sequence_files(data)
kmersize = sailfish.pick_kmersize(files[0])
message = "Creating rapmap {index_type} for {gtf_fa} with {kmersize} bp kmers."
with file_transaction(out_dir) as tx_out_dir:
cmd = "{rapmap} {index_type} -k {kmersize} -i {tx_out_dir} -t {gtf_fa}"
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def summary(*samples):
"""Summarize all quality metrics together"""
samples = utils.unpack_worlds(samples)
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
out_dir = utils.safe_makedir(os.path.join(work_dir, "qc", "mulitqc"))
out_data = os.path.join(out_dir, "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
samples = _report_summary(samples, os.path.join(out_dir, "report"))
if not utils.file_exists(out_file):
with tx_tmpdir(samples[0], work_dir) as tx_out:
in_files = _get_input_files(samples, out_dir, tx_out)
in_files += _merge_metrics(samples, out_dir)
if _one_exists(in_files):
with utils.chdir(out_dir):
_create_config_file(out_dir, samples)
input_list_file = _create_list_file(in_files, out_dir)
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
else:
export_tmp = ""
cmd = "{export_tmp} {multiqc} -f -l {input_list_file} -o {tx_out}"
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
out = []
for i, data in enumerate(_group_by_samplename(samples)):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report", "*.R*"))
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
out.append([data])
return out
def rapmap_pseudoindex(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "pseudoindex", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
rapmap = config_utils.get_program("rapmap", dd.get_config(data))
gtf_fa = sailfish._create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
if file_exists(out_dir + "rapidx.jfhash"):
return out_dir
with file_transaction(out_dir) as tx_out_dir:
cmd = "{rapmap} pseudoindex -k 31 -i {tx_out_dir} -t {gtf_fa}"
message = "Creating rapmap pseudoindex for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
gtf_fa = sailfish._create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
### TODO PUT MEMOZATION HERE
with file_transaction(out_dir) as tx_out_dir:
cmd = "{salmon} index -k 31 -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
gtf_fa = sailfish._create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
### TODO PUT MEMOZATION HERE
with file_transaction(out_dir) as tx_out_dir:
cmd = "{salmon} index -k 31 -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def sailfish_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambiguate(data))
sailfish = config_utils.get_program("sailfish", data["config"])
num_cores = dd.get_num_cores(data)
gtf_fa = _create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
if file_exists(out_dir + "versionInfo.json"):
return out_dir
with file_transaction(out_dir) as tx_out_dir:
cmd = "{sailfish} index -p {num_cores} -t {gtf_fa} -o {tx_out_dir} -k 25"
message = "Creating sailfish index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def sailfish_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambiguate(data))
sailfish = config_utils.get_program("sailfish", data["config"])
num_cores = dd.get_num_cores(data)
gtf_fa = _create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
if file_exists(out_dir + "versionInfo.json"):
return out_dir
with file_transaction(out_dir) as tx_out_dir:
cmd = "{sailfish} index -p {num_cores} -t {gtf_fa} -o {tx_out_dir} -k 25"
message = "Creating sailfish index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def rapmap_index(gtf_file, ref_file, algorithm, data, out_dir):
valid_indexes = ["pseudoindex", "quasiindex"]
index_type = algorithm + "index"
assert index_type in valid_indexes, \
"RapMap only supports %s indices." % valid_indexes
out_dir = os.path.join(out_dir, index_type, dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
rapmap = config_utils.get_program("rapmap", dd.get_config(data))
gtf_fa = sailfish.create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
if file_exists(out_dir + "rapidx.jfhash"):
return out_dir
with file_transaction(out_dir) as tx_out_dir:
cmd = "{rapmap} {index_type} -k 31 -i {tx_out_dir} -t {gtf_fa}"
message = "Creating rapmap {index_type} for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
if dd.get_transcriptome_fasta(data):
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
out_file = os.path.join(out_dir, "versionInfo.json")
if file_exists(out_file):
return out_dir
with file_transaction(out_dir) as tx_out_dir:
cmd = "{salmon} index -k 31 -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", dd.get_genome_build(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
if dd.get_transcriptome_fasta(data):
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data, out_dir)
tmpdir = dd.get_tmp_dir(data)
out_file = os.path.join(out_dir, "versionInfo.json")
if file_exists(out_file):
return out_dir
with file_transaction(out_dir) as tx_out_dir:
cmd = "{salmon} index -k 31 -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa}."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", sailfish.get_build_string(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambiguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
if dd.get_transcriptome_fasta(data):
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data)
assert file_exists(gtf_fa), "%s was not found, exiting." % gtf_fa
tmpdir = dd.get_tmp_dir(data)
out_file = os.path.join(out_dir, "versionInfo.json")
if file_exists(out_file):
return out_dir
files = dd.get_input_sequence_files(data)
kmersize = sailfish.pick_kmersize(files[0])
with file_transaction(data, out_dir) as tx_out_dir:
cmd = "{salmon} index --keepDuplicates -k {kmersize} -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa} with {kmersize} bp kmers."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def salmon_index(gtf_file, ref_file, data, out_dir):
out_dir = os.path.join(out_dir, "index", sailfish.get_build_string(data))
if dd.get_disambiguate(data):
out_dir = "-".join([out_dir] + dd.get_disambiguate(data))
salmon = config_utils.get_program("salmon", dd.get_config(data))
num_cores = dd.get_num_cores(data)
if dd.get_transcriptome_fasta(data):
gtf_fa = dd.get_transcriptome_fasta(data)
else:
gtf_fa = sailfish.create_combined_fasta(data)
assert file_exists(gtf_fa), "%s was not found, exiting." % gtf_fa
tmpdir = dd.get_tmp_dir(data)
out_file = os.path.join(out_dir, "versionInfo.json")
if file_exists(out_file):
logger.info("Transcriptome index for %s detected, skipping building." % gtf_fa)
return out_dir
files = dd.get_input_sequence_files(data)
kmersize = sailfish.pick_kmersize(files[0])
with file_transaction(data, out_dir) as tx_out_dir:
cmd = "{salmon} index -k {kmersize} -p {num_cores} -i {tx_out_dir} -t {gtf_fa}"
message = "Creating Salmon index for {gtf_fa} with {kmersize} bp kmers."
do.run(cmd.format(**locals()), message.format(**locals()), None)
return out_dir
def summary(*samples):
"""Summarize all quality metrics together"""
samples = list(utils.flatten(samples))
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug("multiqc not found. Update bcbio_nextgen.py tools to fix this issue.")
out_dir = utils.safe_makedir(os.path.join(work_dir, "qc", "multiqc"))
out_data = os.path.join(out_dir, "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
file_list = os.path.join(out_dir, "list_files.txt")
work_samples = cwlutils.unpack_tarballs([utils.deepish_copy(x) for x in samples], samples[0])
work_samples = _summarize_inputs(work_samples, out_dir)
if not utils.file_exists(out_file):
with tx_tmpdir(samples[0], work_dir) as tx_out:
in_files = _get_input_files(work_samples, out_dir, tx_out)
in_files += _merge_metrics(work_samples, out_dir)
if _one_exists(in_files):
with utils.chdir(out_dir):
_create_config_file(out_dir, work_samples)
input_list_file = _create_list_file(in_files, file_list)
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s && " % dd.get_tmp_dir(samples[0])
else:
export_tmp = ""
locale_export = utils.locale_export()
path_export = utils.local_path_export()
other_opts = config_utils.get_resources("multiqc", samples[0]["config"]).get("options", [])
other_opts = " ".join([str(x) for x in other_opts])
cmd = ("{path_export}{export_tmp}{locale_export} "
"{multiqc} -f -l {input_list_file} {other_opts} -o {tx_out}")
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(os.path.join(tx_out, "multiqc_report.html")):
shutil.move(os.path.join(tx_out, "multiqc_report.html"), out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"), out_data)
samples = _group_by_sample_and_batch(samples)
if utils.file_exists(out_file) and samples:
data_files = set()
for i, data in enumerate(samples):
data_files.add(os.path.join(out_dir, "report", "metrics", dd.get_sample_name(data) + "_bcbio.txt"))
data_files.add(os.path.join(out_dir, "report", "metrics", "target_info.yaml"))
data_files.add(os.path.join(out_dir, "multiqc_config.yaml"))
[data_files.add(f) for f in glob.glob(os.path.join(out_dir, "multiqc_data", "*"))]
data_files = [f for f in data_files if f and utils.file_exists(f)]
if "summary" not in samples[0]:
samples[0]["summary"] = {}
samples[0]["summary"]["multiqc"] = {"base": out_file, "secondary": data_files}
data_json = os.path.join(out_dir, "multiqc_data", "multiqc_data.json")
data_json_final = _save_uploaded_data_json(samples, data_json, os.path.join(out_dir, "multiqc_data"))
if data_json_final:
samples[0]["summary"]["multiqc"]["secondary"].append(data_json_final)
# Prepare final file list and inputs for downstream usage
file_list_final = _save_uploaded_file_list(samples, file_list, out_dir)
if file_list_final:
samples[0]["summary"]["multiqc"]["secondary"].append(file_list_final)
if any([cwlutils.is_cwl_run(d) for d in samples]):
for indir in ["inputs", "report"]:
tarball = os.path.join(out_dir, "multiqc-%s.tar.gz" % (indir))
if not utils.file_exists(tarball):
with utils.chdir(out_dir):
cmd = ["tar", "-czvpf", tarball, indir]
do.run(cmd, "Compress multiqc inputs: %s" % indir)
samples[0]["summary"]["multiqc"]["secondary"].append(tarball)
if any([cwlutils.is_cwl_run(d) for d in samples]):
samples = _add_versions(samples)
return [[data] for data in samples]
def summary(*samples):
"""Summarize all quality metrics together"""
samples = utils.unpack_worlds(samples)
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug(
"multiqc not found. Update bcbio_nextgen.py tools to fix this issue."
)
file_fapths = []
opts = ""
out_dir = os.path.join(work_dir, "multiqc")
out_data = os.path.join(work_dir, "multiqc", "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
samples = _report_summary(samples, os.path.join(out_dir, "report"))
for data in samples:
for program, pfiles in tz.get_in(["summary", "qc"], data,
{}).iteritems():
if isinstance(pfiles, dict):
pfiles = [pfiles["base"]] + pfiles["secondary"]
elif isinstance(pfiles, basestring):
pfiles = [pfiles]
file_fapths.extend(pfiles)
file_fapths.append(
os.path.join(out_dir, "report", "metrics", "target_info.yaml"))
# XXX temporary workaround until we can handle larger inputs through MultiQC
file_fapths = list(set(file_fapths))
# Back compatible -- to migrate to explicit specifications in input YAML
file_fapths += ["trimmed", "htseq-count/*summary"]
if not utils.file_exists(out_file):
with utils.chdir(work_dir):
file_fapths = [
fpath for fpath in file_fapths if _check_multiqc_input(fpath)
and _is_good_file_for_multiqc(fpath)
]
input_list_file = _create_list_file(file_fapths)
export_tmp = ""
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
if input_list_file:
cmd = "{export_tmp} {multiqc} -f -l {input_list_file} -o {tx_out} {opts}"
with tx_tmpdir(data, work_dir) as tx_out:
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(
os.path.join(tx_out, "multiqc_report.html")):
shutil.move(
os.path.join(tx_out, "multiqc_report.html"),
out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"),
out_data)
out = []
for i, data in enumerate(samples):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(
os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(
os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(
os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(
os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report",
"*.R*"))
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {
"base": out_file,
"secondary": data_files
}
out.append(data)
return [[fpath] for fpath in out]
def summary(*samples):
"""Summarize all quality metrics together"""
samples = utils.unpack_worlds(samples)
work_dir = dd.get_work_dir(samples[0])
multiqc = config_utils.get_program("multiqc", samples[0]["config"])
if not multiqc:
logger.debug(
"multiqc not found. Update bcbio_nextgen.py tools to fix this issue."
)
folders = []
opts = ""
out_dir = os.path.join(work_dir, "multiqc")
out_data = os.path.join(work_dir, "multiqc", "multiqc_data")
out_file = os.path.join(out_dir, "multiqc_report.html")
samples = _report_summary(samples, os.path.join(out_dir, "report"))
for data in samples:
for program, pfiles in tz.get_in(["summary", "qc"], data,
{}).iteritems():
if isinstance(pfiles, dict):
pfiles = pfiles["base"]
folders.append(os.path.dirname(pfiles))
# XXX temporary workaround until we can handle larger inputs through MultiQC
folders = list(set(folders))
if len(folders) > 250:
logger.warning(
"Too many samples for MultiQC, only using first 250 entries.")
folders = folders[:250]
opts = "--flat"
# Back compatible -- to migrate to explicit specifications in input YAML
folders += ["trimmed", "htseq-count/*summary"]
if not utils.file_exists(out_file):
with utils.chdir(work_dir):
input_dir = " ".join([_check_multiqc_input(d) for d in folders])
export_tmp = ""
if dd.get_tmp_dir(samples[0]):
export_tmp = "export TMPDIR=%s &&" % dd.get_tmp_dir(samples[0])
if input_dir.strip():
cmd = "{export_tmp} {multiqc} -f {input_dir} -o {tx_out} {opts}"
with tx_tmpdir(data, work_dir) as tx_out:
do.run(cmd.format(**locals()), "Run multiqc")
if utils.file_exists(
os.path.join(tx_out, "multiqc_report.html")):
shutil.move(
os.path.join(tx_out, "multiqc_report.html"),
out_file)
shutil.move(os.path.join(tx_out, "multiqc_data"),
out_data)
out = []
for i, data in enumerate(samples):
if i == 0:
if utils.file_exists(out_file):
data_files = glob.glob(
os.path.join(out_dir, "multiqc_data", "*.txt"))
data_files += glob.glob(
os.path.join(out_dir, "report", "*", "*.bed"))
data_files += glob.glob(
os.path.join(out_dir, "report", "*", "*.txt"))
data_files += glob.glob(
os.path.join(out_dir, "report", "*", "*.tsv"))
data_files += glob.glob(os.path.join(out_dir, "report",
"*.R*"))
if "summary" not in data:
data["summary"] = {}
data["summary"]["multiqc"] = {
"base": out_file,
"secondary": data_files
}
out.append(data)
return [[d] for d in out]