def _get_files_project(sample, upload_config):
"""Retrieve output files associated with an entire analysis project.
"""
out = [{"path": sample["provenance"]["programs"]}]
for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
"type": "external_command_log",
"ext": ""})
if "summary" in sample and sample["summary"].get("project"):
out.append({"path": sample["summary"]["project"]})
mixup_check = tz.get_in(["summary", "mixup_check"], sample)
if mixup_check:
out.append({"path": sample["summary"]["mixup_check"],
"type": "directory", "ext": "mixup_check"})
for x in sample.get("variants", []):
if "pop_db" in x:
out.append({"path": x["pop_db"],
"type": "sqlite",
"variantcaller": x["variantcaller"]})
for x in sample.get("variants", []):
if "population" in x:
pop_db = tz.get_in(["population", "db"], x)
if pop_db:
out.append({"path": pop_db,
"type": "sqlite",
"variantcaller": x["variantcaller"]})
out.extend(_get_variant_file(x, ("population", "vcf")))
for x in sample.get("variants", []):
if x.get("validate") and x["validate"].get("grading_summary"):
out.append({"path": x["validate"]["grading_summary"]})
break
if "coverage" in sample:
cov_db = tz.get_in(["coverage", "summary"], sample)
if cov_db:
out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
if "combined_counts" in sample:
out.append({"path": sample["combined_counts"]})
if "annotated_combined_counts" in sample:
out.append({"path": sample["annotated_combined_counts"]})
if "combined_fpkm" in sample:
out.append({"path": sample["combined_fpkm"]})
if "combined_fpkm_isoform" in sample:
out.append({"path": sample["combined_fpkm_isoform"]})
if "assembled_gtf" in sample:
out.append({"path": sample["assembled_gtf"]})
if dd.get_dexseq_counts(sample):
out.append({"path": dd.get_dexseq_counts(sample)})
if dd.get_express_counts(sample):
out.append({"path": dd.get_express_counts(sample)})
if dd.get_express_fpkm(sample):
out.append({"path": dd.get_express_fpkm(sample)})
if dd.get_express_tpm(sample):
out.append({"path": dd.get_express_tpm(sample)})
return _add_meta(out, config=upload_config)
def combine_express(samples, combined):
"""Combine tpm, effective counts and fpkm from express results"""
if not combined:
return None
to_combine = [
dd.get_express_counts(x) for x in dd.sample_data_iterator(samples)
if dd.get_express_counts(x)
]
gtf_file = dd.get_gtf_file(samples[0][0])
isoform_to_gene_file = os.path.join(os.path.dirname(combined),
"isoform_to_gene.txt")
isoform_to_gene_file = express.isoform_to_gene_name(
gtf_file, isoform_to_gene_file,
dd.sample_data_iterator(samples).next())
if len(to_combine) > 0:
eff_counts_combined_file = os.path.splitext(
combined)[0] + ".isoform.express_counts"
eff_counts_combined = count.combine_count_files(
to_combine, eff_counts_combined_file, ext=".counts")
to_combine = [
dd.get_express_tpm(x) for x in dd.sample_data_iterator(samples)
if dd.get_express_tpm(x)
]
tpm_counts_combined_file = os.path.splitext(
combined)[0] + ".isoform.express_tpm"
tpm_counts_combined = count.combine_count_files(
to_combine, tpm_counts_combined_file)
to_combine = [
dd.get_express_fpkm(x) for x in dd.sample_data_iterator(samples)
if dd.get_express_fpkm(x)
]
fpkm_counts_combined_file = os.path.splitext(
combined)[0] + ".isoform.express_fpkm"
fpkm_counts_combined = count.combine_count_files(
to_combine, fpkm_counts_combined_file, ext=".fpkm")
return {
'counts': eff_counts_combined,
'tpm': tpm_counts_combined,
'fpkm': fpkm_counts_combined,
'isoform_to_gene': isoform_to_gene_file
}
return {}
def combine_express(samples, combined):
"""Combine tpm, effective counts and fpkm from express results"""
to_combine = [dd.get_express_counts(x) for x in
dd.sample_data_iterator(samples) if dd.get_express_counts(x)]
gtf_file = dd.get_gtf_file(samples[0][0])
isoform_to_gene_file = os.path.join(os.path.dirname(combined), "isoform_to_gene.txt")
isoform_to_gene_file = express.isoform_to_gene_name(gtf_file, isoform_to_gene_file)
if len(to_combine) > 0:
eff_counts_combined_file = os.path.splitext(combined)[0] + ".isoform.express_counts"
eff_counts_combined = count.combine_count_files(to_combine, eff_counts_combined_file)
to_combine = [dd.get_express_tpm(x) for x in
dd.sample_data_iterator(samples) if dd.get_express_tpm(x)]
tpm_counts_combined_file = os.path.splitext(combined)[0] + ".isoform.express_tpm"
tpm_counts_combined = count.combine_count_files(to_combine, tpm_counts_combined_file)
to_combine = [dd.get_express_fpkm(x) for x in dd.sample_data_iterator(samples)
if dd.get_express_fpkm(x)]
fpkm_counts_combined_file = os.path.splitext(combined)[0] + ".isoform.express_fpkm"
fpkm_counts_combined = count.combine_count_files(to_combine, fpkm_counts_combined_file)
return {'counts': eff_counts_combined, 'tpm': tpm_counts_combined,
'fpkm': fpkm_counts_combined, 'isoform_to_gene': isoform_to_gene_file}
return {}
def _get_files_project(sample, upload_config):
"""Retrieve output files associated with an entire analysis project.
"""
out = [{"path": sample["provenance"]["programs"]}]
if os.path.exists(tz.get_in(["provenance", "data"], sample) or ""):
out.append({"path": sample["provenance"]["data"]})
for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
if os.path.exists(
os.path.join(log.get_log_dir(sample["config"]), fname)):
out.append({
"path":
os.path.join(log.get_log_dir(sample["config"]), fname),
"type":
"external_command_log",
"ext":
""
})
if "summary" in sample and sample["summary"].get("project"):
out.append({"path": sample["summary"]["project"]})
if "summary" in sample and sample["summary"].get("metadata"):
out.append({"path": sample["summary"]["metadata"]})
mixup_check = tz.get_in(["summary", "mixup_check"], sample)
if mixup_check:
out.append({
"path": sample["summary"]["mixup_check"],
"type": "directory",
"ext": "mixup_check"
})
report = os.path.join(dd.get_work_dir(sample), "report")
if utils.file_exists(report):
out.append({"path": report, "type": "directory", "ext": "report"})
multiqc = tz.get_in(["summary", "multiqc"], sample)
if multiqc:
out.extend(_flatten_file_with_secondary(multiqc, "multiqc"))
if sample.get("seqcluster", {}):
out.append({
"path": sample["seqcluster"].get("out_dir"),
"type": "directory",
"ext": "seqcluster"
})
if sample.get("mirge", {}):
for fn in sample["mirge"]:
out.append({"path": fn, "dir": "mirge"})
if sample.get("report", None):
out.append({
"path": os.path.dirname(sample["report"]),
"type": "directory",
"ext": "seqclusterViz"
})
for x in sample.get("variants", []):
if "pop_db" in x:
out.append({
"path": x["pop_db"],
"type": "sqlite",
"variantcaller": x["variantcaller"]
})
for x in sample.get("variants", []):
if "population" in x:
pop_db = tz.get_in(["population", "db"], x)
if pop_db:
out.append({
"path": pop_db,
"type": "sqlite",
"variantcaller": x["variantcaller"]
})
suffix = "-annotated-decomposed" if tz.get_in(
("population", "decomposed"), x) else "-annotated"
vcfs = _get_project_vcf(x, suffix)
out.extend([_add_batch(f, sample) for f in vcfs])
for x in sample.get("variants", []):
if x.get("validate") and x["validate"].get("grading_summary"):
out.append({"path": x["validate"]["grading_summary"]})
break
sv_project = set([])
for svcall in sample.get("sv", []):
if svcall.get("variantcaller") == "seq2c":
if svcall.get(
"calls_all") and svcall["calls_all"] not in sv_project:
out.append({
"path": svcall["coverage_all"],
"batch": "seq2c",
"ext": "coverage",
"type": "tsv"
})
out.append({
"path": svcall["read_mapping"],
"batch": "seq2c",
"ext": "read_mapping",
"type": "txt"
})
out.append({
"path": svcall["calls_all"],
"batch": "seq2c",
"ext": "calls",
"type": "tsv"
})
sv_project.add(svcall["calls_all"])
if "coverage" in sample:
cov_db = tz.get_in(["coverage", "summary"], sample)
if cov_db:
out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
all_coverage = tz.get_in(["coverage", "all"], sample)
if all_coverage:
out.append({
"path": all_coverage,
"type": "bed",
"ext": "coverage"
})
if dd.get_mirna_counts(sample):
out.append({"path": dd.get_mirna_counts(sample)})
if dd.get_isomir_counts(sample):
out.append({"path": dd.get_isomir_counts(sample)})
if dd.get_novel_mirna_counts(sample):
out.append({"path": dd.get_novel_mirna_counts(sample)})
if dd.get_novel_isomir_counts(sample):
out.append({"path": dd.get_novel_isomir_counts(sample)})
if dd.get_combined_counts(sample):
count_file = dd.get_combined_counts(sample)
if sample["analysis"].lower() == "scrna-seq":
out.append({"path": count_file, "type": "mtx"})
out.append({"path": count_file + ".rownames", "type": "rownames"})
out.append({"path": count_file + ".colnames", "type": "colnames"})
out.append({"path": count_file + ".metadata", "type": "metadata"})
umi_file = os.path.splitext(count_file)[0] + "-dupes.mtx"
if utils.file_exists(umi_file):
out.append({"path": umi_file, "type": "mtx"})
out.append({
"path": umi_file + ".rownames",
"type": "rownames"
})
out.append({
"path": umi_file + ".colnames",
"type": "colnames"
})
if dd.get_combined_histogram(sample):
out.append({
"path": dd.get_combined_histogram(sample),
"type": "txt"
})
rda = os.path.join(os.path.dirname(count_file), "se.rda")
if utils.file_exists(rda):
out.append({"path": rda, "type": "rda"})
else:
out.append({"path": dd.get_combined_counts(sample)})
if dd.get_tximport(sample):
out.append({"path": dd.get_tximport(sample)["gene_tpm"], "dir": "tpm"})
out.append({
"path": dd.get_tximport(sample)["gene_counts"],
"dir": "counts"
})
if dd.get_annotated_combined_counts(sample):
out.append({"path": dd.get_annotated_combined_counts(sample)})
if dd.get_combined_fpkm(sample):
out.append({"path": dd.get_combined_fpkm(sample)})
if dd.get_combined_fpkm_isoform(sample):
out.append({"path": dd.get_combined_fpkm_isoform(sample)})
if dd.get_transcript_assembler(sample):
out.append({"path": dd.get_merged_gtf(sample)})
if dd.get_dexseq_counts(sample):
out.append({"path": dd.get_dexseq_counts(sample)})
out.append({"path": "%s.ann" % dd.get_dexseq_counts(sample)})
if dd.get_express_counts(sample):
out.append({"path": dd.get_express_counts(sample)})
if dd.get_express_fpkm(sample):
out.append({"path": dd.get_express_fpkm(sample)})
if dd.get_express_tpm(sample):
out.append({"path": dd.get_express_tpm(sample)})
if dd.get_isoform_to_gene(sample):
out.append({"path": dd.get_isoform_to_gene(sample)})
if dd.get_square_vcf(sample):
out.append({"path": dd.get_square_vcf(sample)})
if dd.get_sailfish_transcript_tpm(sample):
out.append({"path": dd.get_sailfish_transcript_tpm(sample)})
if dd.get_sailfish_gene_tpm(sample):
out.append({"path": dd.get_sailfish_gene_tpm(sample)})
if dd.get_tx2gene(sample):
out.append({"path": dd.get_tx2gene(sample)})
if dd.get_spikein_counts(sample):
out.append({"path": dd.get_spikein_counts(sample)})
if tz.get_in(("peaks_files", "consensus", "main"), sample):
out.append({
"path":
tz.get_in(("peaks_files", "consensus", "main"), sample),
"dir":
"consensus"
})
if tz.get_in(("peak_counts", "peaktable"), sample):
out.append({
"path": tz.get_in(("peak_counts", "peaktable"), sample),
"dir": "consensus"
})
transcriptome_dir = os.path.join(dd.get_work_dir(sample), "inputs",
"transcriptome")
if os.path.exists(transcriptome_dir):
out.append({
"path": transcriptome_dir,
"type": "directory",
"ext": "transcriptome"
})
return _add_meta(out, config=upload_config)
def _get_files_project(sample, upload_config):
"""Retrieve output files associated with an entire analysis project.
"""
out = [{"path": sample["provenance"]["programs"]}]
if os.path.exists(tz.get_in(["provenance", "data"], sample) or ""):
out.append({"path": sample["provenance"]["data"]})
for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
"type": "external_command_log",
"ext": ""})
if "summary" in sample and sample["summary"].get("project"):
out.append({"path": sample["summary"]["project"]})
mixup_check = tz.get_in(["summary", "mixup_check"], sample)
if mixup_check:
out.append({"path": sample["summary"]["mixup_check"],
"type": "directory", "ext": "mixup_check"})
report = os.path.join(dd.get_work_dir(sample), "report")
if utils.file_exists(report):
out.append({"path": report,
"type": "directory", "ext": "report"})
multiqc = tz.get_in(["summary", "multiqc"], sample)
if multiqc:
out.extend(_flatten_file_with_secondary(multiqc, "multiqc"))
if sample.get("seqcluster", None):
out.append({"path": sample["seqcluster"],
"type": "directory", "ext": "seqcluster"})
if sample.get("report", None):
out.append({"path": os.path.dirname(sample["report"]),
"type": "directory", "ext": "seqclusterViz"})
for x in sample.get("variants", []):
if "pop_db" in x:
out.append({"path": x["pop_db"],
"type": "sqlite",
"variantcaller": x["variantcaller"]})
for x in sample.get("variants", []):
if "population" in x:
pop_db = tz.get_in(["population", "db"], x)
if pop_db:
out.append({"path": pop_db,
"type": "sqlite",
"variantcaller": x["variantcaller"]})
out.extend(_get_variant_file(x, ("population", "vcf")))
for x in sample.get("variants", []):
if x.get("validate") and x["validate"].get("grading_summary"):
out.append({"path": x["validate"]["grading_summary"]})
break
if "coverage" in sample:
cov_db = tz.get_in(["coverage", "summary"], sample)
if cov_db:
out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
all_coverage = tz.get_in(["coverage", "all"], sample)
if all_coverage:
out.append({"path": all_coverage, "type": "bed", "ext": "coverage"})
if dd.get_mirna_counts(sample):
out.append({"path": dd.get_mirna_counts(sample)})
if dd.get_isomir_counts(sample):
out.append({"path": dd.get_isomir_counts(sample)})
if dd.get_novel_mirna_counts(sample):
out.append({"path": dd.get_novel_mirna_counts(sample)})
if dd.get_novel_isomir_counts(sample):
out.append({"path": dd.get_novel_isomir_counts(sample)})
if dd.get_combined_counts(sample):
out.append({"path": dd.get_combined_counts(sample)})
if dd.get_annotated_combined_counts(sample):
out.append({"path": dd.get_annotated_combined_counts(sample)})
if dd.get_combined_fpkm(sample):
out.append({"path": dd.get_combined_fpkm(sample)})
if dd.get_combined_fpkm_isoform(sample):
out.append({"path": dd.get_combined_fpkm_isoform(sample)})
if dd.get_transcript_assembler(sample):
out.append({"path": dd.get_merged_gtf(sample)})
if dd.get_dexseq_counts(sample):
out.append({"path": dd.get_dexseq_counts(sample)})
if dd.get_express_counts(sample):
out.append({"path": dd.get_express_counts(sample)})
if dd.get_express_fpkm(sample):
out.append({"path": dd.get_express_fpkm(sample)})
if dd.get_express_tpm(sample):
out.append({"path": dd.get_express_tpm(sample)})
if dd.get_isoform_to_gene(sample):
out.append({"path": dd.get_isoform_to_gene(sample)})
if dd.get_square_vcf(sample):
out.append({"path": dd.get_square_vcf(sample)})
if dd.get_sailfish_tidy(sample):
out.append({"path": dd.get_sailfish_tidy(sample)})
if dd.get_sailfish_transcript_tpm(sample):
out.append({"path": dd.get_sailfish_transcript_tpm(sample)})
if dd.get_sailfish_gene_tpm(sample):
out.append({"path": dd.get_sailfish_gene_tpm(sample)})
if dd.get_tx2gene(sample):
out.append({"path": dd.get_tx2gene(sample)})
return _add_meta(out, config=upload_config)
def _get_files_project(sample, upload_config):
"""Retrieve output files associated with an entire analysis project.
"""
out = [{"path": sample["provenance"]["programs"]}]
for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
"type": "external_command_log",
"ext": ""})
if "summary" in sample and sample["summary"].get("project"):
out.append({"path": sample["summary"]["project"]})
mixup_check = tz.get_in(["summary", "mixup_check"], sample)
if mixup_check:
out.append({"path": sample["summary"]["mixup_check"],
"type": "directory", "ext": "mixup_check"})
if sample.get("seqcluster", None):
out.append({"path": sample["seqcluster"],
"type": "directory", "ext": "seqcluster"})
for x in sample.get("variants", []):
if "pop_db" in x:
out.append({"path": x["pop_db"],
"type": "sqlite",
"variantcaller": x["variantcaller"]})
for x in sample.get("variants", []):
if "population" in x:
pop_db = tz.get_in(["population", "db"], x)
if pop_db:
out.append({"path": pop_db,
"type": "sqlite",
"variantcaller": x["variantcaller"]})
out.extend(_get_variant_file(x, ("population", "vcf")))
for x in sample.get("variants", []):
if x.get("validate") and x["validate"].get("grading_summary"):
out.append({"path": x["validate"]["grading_summary"]})
break
if "coverage" in sample:
cov_db = tz.get_in(["coverage", "summary"], sample)
if cov_db:
out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
all_coverage = tz.get_in(["coverage", "all"], sample)
if all_coverage:
out.append({"path": all_coverage, "type": "bed", "ext": "coverage"})
if dd.get_combined_counts(sample):
out.append({"path": dd.get_combined_counts(sample)})
if dd.get_annotated_combined_counts(sample):
out.append({"path": dd.get_annotated_combined_counts(sample)})
if dd.get_combined_fpkm(sample):
out.append({"path": dd.get_combined_fpkm(sample)})
if dd.get_combined_fpkm_isoform(sample):
out.append({"path": dd.get_combined_fpkm_isoform(sample)})
if dd.get_assembled_gtf(sample):
out.append({"path": dd.get_assembled_gtf(sample)})
if dd.get_dexseq_counts(sample):
out.append({"path": dd.get_dexseq_counts(sample)})
if dd.get_express_counts(sample):
out.append({"path": dd.get_express_counts(sample)})
if dd.get_express_fpkm(sample):
out.append({"path": dd.get_express_fpkm(sample)})
if dd.get_express_tpm(sample):
out.append({"path": dd.get_express_tpm(sample)})
if dd.get_isoform_to_gene(sample):
out.append({"path": dd.get_isoform_to_gene(sample)})
if dd.get_square_vcf(sample):
out.append({"path": dd.get_square_vcf(sample)})
return _add_meta(out, config=upload_config)
def _get_files_project(sample, upload_config):
"""Retrieve output files associated with an entire analysis project.
"""
out = [{"path": sample["provenance"]["programs"]}]
if os.path.exists(tz.get_in(["provenance", "data"], sample) or ""):
out.append({"path": sample["provenance"]["data"]})
for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
"type": "external_command_log",
"ext": ""})
if "summary" in sample and sample["summary"].get("project"):
out.append({"path": sample["summary"]["project"]})
mixup_check = tz.get_in(["summary", "mixup_check"], sample)
if mixup_check:
out.append({"path": sample["summary"]["mixup_check"],
"type": "directory", "ext": "mixup_check"})
report = os.path.join(dd.get_work_dir(sample), "report")
if utils.file_exists(report):
out.append({"path": report,
"type": "directory", "ext": "report"})
multiqc = tz.get_in(["summary", "multiqc"], sample)
if multiqc:
out.extend(_flatten_file_with_secondary(multiqc, "multiqc"))
if sample.get("seqcluster", {}):
out.append({"path": sample["seqcluster"].get("out_dir"),
"type": "directory", "ext": "seqcluster"})
if sample.get("report", None):
out.append({"path": os.path.dirname(sample["report"]),
"type": "directory", "ext": "seqclusterViz"})
for x in sample.get("variants", []):
if "pop_db" in x:
out.append({"path": x["pop_db"],
"type": "sqlite",
"variantcaller": x["variantcaller"]})
for x in sample.get("variants", []):
if "population" in x:
pop_db = tz.get_in(["population", "db"], x)
if pop_db:
out.append({"path": pop_db,
"type": "sqlite",
"variantcaller": x["variantcaller"]})
suffix = "-annotated-decomposed" if tz.get_in(("population", "decomposed"), x) else "-annotated"
out.extend([_add_batch(x, sample)
for x in _get_variant_file(x, ("population", "vcf"), suffix=suffix)])
for x in sample.get("variants", []):
if x.get("validate") and x["validate"].get("grading_summary"):
out.append({"path": x["validate"]["grading_summary"]})
break
if "coverage" in sample:
cov_db = tz.get_in(["coverage", "summary"], sample)
if cov_db:
out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
all_coverage = tz.get_in(["coverage", "all"], sample)
if all_coverage:
out.append({"path": all_coverage, "type": "bed", "ext": "coverage"})
if dd.get_mirna_counts(sample):
out.append({"path": dd.get_mirna_counts(sample)})
if dd.get_isomir_counts(sample):
out.append({"path": dd.get_isomir_counts(sample)})
if dd.get_novel_mirna_counts(sample):
out.append({"path": dd.get_novel_mirna_counts(sample)})
if dd.get_novel_isomir_counts(sample):
out.append({"path": dd.get_novel_isomir_counts(sample)})
if dd.get_combined_counts(sample):
count_file = dd.get_combined_counts(sample)
if sample["analysis"].lower() == "scrna-seq":
out.append({"path": count_file,
"type": "mtx"})
out.append({"path": count_file + ".rownames",
"type": "rownames"})
out.append({"path": count_file + ".colnames",
"type": "colnames"})
else:
out.append({"path": dd.get_combined_counts(sample)})
if dd.get_annotated_combined_counts(sample):
out.append({"path": dd.get_annotated_combined_counts(sample)})
if dd.get_combined_fpkm(sample):
out.append({"path": dd.get_combined_fpkm(sample)})
if dd.get_combined_fpkm_isoform(sample):
out.append({"path": dd.get_combined_fpkm_isoform(sample)})
if dd.get_transcript_assembler(sample):
out.append({"path": dd.get_merged_gtf(sample)})
if dd.get_dexseq_counts(sample):
out.append({"path": dd.get_dexseq_counts(sample)})
if dd.get_express_counts(sample):
out.append({"path": dd.get_express_counts(sample)})
if dd.get_express_fpkm(sample):
out.append({"path": dd.get_express_fpkm(sample)})
if dd.get_express_tpm(sample):
out.append({"path": dd.get_express_tpm(sample)})
if dd.get_isoform_to_gene(sample):
out.append({"path": dd.get_isoform_to_gene(sample)})
if dd.get_square_vcf(sample):
out.append({"path": dd.get_square_vcf(sample)})
if dd.get_sailfish_tidy(sample):
out.append({"path": dd.get_sailfish_tidy(sample)})
if dd.get_sailfish_transcript_tpm(sample):
out.append({"path": dd.get_sailfish_transcript_tpm(sample)})
if dd.get_sailfish_gene_tpm(sample):
out.append({"path": dd.get_sailfish_gene_tpm(sample)})
if dd.get_tx2gene(sample):
out.append({"path": dd.get_tx2gene(sample)})
if dd.get_spikein_counts(sample):
out.append({"path": dd.get_spikein_counts(sample)})
return _add_meta(out, config=upload_config)
def _get_files_project(sample, upload_config):
"""Retrieve output files associated with an entire analysis project.
"""
out = [{"path": sample["provenance"]["programs"]}]
if os.path.exists(tz.get_in(["provenance", "data"], sample) or ""):
out.append({"path": sample["provenance"]["data"]})
for fname in ["bcbio-nextgen.log", "bcbio-nextgen-commands.log"]:
if os.path.exists(os.path.join(log.get_log_dir(sample["config"]), fname)):
out.append({"path": os.path.join(log.get_log_dir(sample["config"]), fname),
"type": "external_command_log",
"ext": ""})
if "summary" in sample and sample["summary"].get("project"):
out.append({"path": sample["summary"]["project"]})
if "summary" in sample and sample["summary"].get("metadata"):
out.append({"path": sample["summary"]["metadata"]})
mixup_check = tz.get_in(["summary", "mixup_check"], sample)
if mixup_check:
out.append({"path": sample["summary"]["mixup_check"],
"type": "directory", "ext": "mixup_check"})
report = os.path.join(dd.get_work_dir(sample), "report")
if utils.file_exists(report):
out.append({"path": report,
"type": "directory", "ext": "report"})
multiqc = tz.get_in(["summary", "multiqc"], sample)
if multiqc:
out.extend(_flatten_file_with_secondary(multiqc, "multiqc"))
if sample.get("seqcluster", {}):
out.append({"path": sample["seqcluster"].get("out_dir"),
"type": "directory", "ext": "seqcluster"})
if sample.get("mirge", {}):
for fn in sample["mirge"]:
out.append({"path": fn,
"dir": "mirge"})
if sample.get("report", None):
out.append({"path": os.path.dirname(sample["report"]),
"type": "directory", "ext": "seqclusterViz"})
for x in sample.get("variants", []):
if "pop_db" in x:
out.append({"path": x["pop_db"],
"type": "sqlite",
"variantcaller": x["variantcaller"]})
for x in sample.get("variants", []):
if "population" in x:
pop_db = tz.get_in(["population", "db"], x)
if pop_db:
out.append({"path": pop_db,
"type": "sqlite",
"variantcaller": x["variantcaller"]})
suffix = "-annotated-decomposed" if tz.get_in(("population", "decomposed"), x) else "-annotated"
vcfs = _get_project_vcf(x, suffix)
out.extend([_add_batch(f, sample) for f in vcfs])
for x in sample.get("variants", []):
if x.get("validate") and x["validate"].get("grading_summary"):
out.append({"path": x["validate"]["grading_summary"]})
break
sv_project = set([])
for svcall in sample.get("sv", []):
if svcall.get("variantcaller") == "seq2c":
if svcall.get("calls_all") and svcall["calls_all"] not in sv_project:
out.append({"path": svcall["coverage_all"], "batch": "seq2c", "ext": "coverage", "type": "tsv"})
out.append({"path": svcall["read_mapping"], "batch": "seq2c", "ext": "read_mapping", "type": "txt"})
out.append({"path": svcall["calls_all"], "batch": "seq2c", "ext": "calls", "type": "tsv"})
sv_project.add(svcall["calls_all"])
if "coverage" in sample:
cov_db = tz.get_in(["coverage", "summary"], sample)
if cov_db:
out.append({"path": cov_db, "type": "sqlite", "ext": "coverage"})
all_coverage = tz.get_in(["coverage", "all"], sample)
if all_coverage:
out.append({"path": all_coverage, "type": "bed", "ext": "coverage"})
if dd.get_mirna_counts(sample):
out.append({"path": dd.get_mirna_counts(sample)})
if dd.get_isomir_counts(sample):
out.append({"path": dd.get_isomir_counts(sample)})
if dd.get_novel_mirna_counts(sample):
out.append({"path": dd.get_novel_mirna_counts(sample)})
if dd.get_novel_isomir_counts(sample):
out.append({"path": dd.get_novel_isomir_counts(sample)})
if dd.get_combined_counts(sample):
count_file = dd.get_combined_counts(sample)
if sample["analysis"].lower() == "scrna-seq":
out.append({"path": count_file,
"type": "mtx"})
out.append({"path": count_file + ".rownames",
"type": "rownames"})
out.append({"path": count_file + ".colnames",
"type": "colnames"})
out.append({"path": count_file + ".metadata",
"type": "metadata"})
umi_file = os.path.splitext(count_file)[0] + "-dupes.mtx"
if utils.file_exists(umi_file):
out.append({"path": umi_file,
"type": "mtx"})
out.append({"path": umi_file + ".rownames",
"type": "rownames"})
out.append({"path": umi_file + ".colnames",
"type": "colnames"})
if dd.get_combined_histogram(sample):
out.append({"path": dd.get_combined_histogram(sample),
"type": "txt"})
rda = os.path.join(os.path.dirname(count_file), "se.rda")
if utils.file_exists(rda):
out.append({"path": rda,
"type": "rda"})
else:
out.append({"path": dd.get_combined_counts(sample)})
if dd.get_annotated_combined_counts(sample):
out.append({"path": dd.get_annotated_combined_counts(sample)})
if dd.get_combined_fpkm(sample):
out.append({"path": dd.get_combined_fpkm(sample)})
if dd.get_combined_fpkm_isoform(sample):
out.append({"path": dd.get_combined_fpkm_isoform(sample)})
if dd.get_transcript_assembler(sample):
out.append({"path": dd.get_merged_gtf(sample)})
if dd.get_dexseq_counts(sample):
out.append({"path": dd.get_dexseq_counts(sample)})
out.append({"path": "%s.ann" % dd.get_dexseq_counts(sample)})
if dd.get_express_counts(sample):
out.append({"path": dd.get_express_counts(sample)})
if dd.get_express_fpkm(sample):
out.append({"path": dd.get_express_fpkm(sample)})
if dd.get_express_tpm(sample):
out.append({"path": dd.get_express_tpm(sample)})
if dd.get_isoform_to_gene(sample):
out.append({"path": dd.get_isoform_to_gene(sample)})
if dd.get_square_vcf(sample):
out.append({"path": dd.get_square_vcf(sample)})
if dd.get_sailfish_transcript_tpm(sample):
out.append({"path": dd.get_sailfish_transcript_tpm(sample)})
if dd.get_sailfish_gene_tpm(sample):
out.append({"path": dd.get_sailfish_gene_tpm(sample)})
if dd.get_tx2gene(sample):
out.append({"path": dd.get_tx2gene(sample)})
if dd.get_spikein_counts(sample):
out.append({"path": dd.get_spikein_counts(sample)})
transcriptome_dir = os.path.join(dd.get_work_dir(sample), "inputs",
"transcriptome")
if os.path.exists(transcriptome_dir):
out.append({"path": transcriptome_dir, "type": "directory",
"ext": "transcriptome"})
return _add_meta(out, config=upload_config)
