def combine_files(samples):
"""
after quantitation, combine the counts/FPKM/TPM/etc into a single table with
all samples
"""
gtf_file = dd.get_gtf_file(samples[0][0], None)
dexseq_gff = dd.get_dexseq_gff(samples[0][0])
# combine featureCount files
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files, ext=".counts")
annotated = count.annotate_combined_count_file(combined, gtf_file)
# combine eXpress files
express_counts_combined = combine_express(samples, combined)
# combine Cufflinks files
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
if fpkm_files:
fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file)
else:
fpkm_combined = None
fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm"
isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples])
if isoform_files:
fpkm_isoform_combined = count.combine_count_files(isoform_files,
fpkm_isoform_combined_file,
".isoform.fpkm")
else:
fpkm_isoform_combined = None
# combine DEXseq files
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data in samples])
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file, ".dexseq")
dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined)
else:
dexseq_combined = None
samples = spikein.combine_spikein(samples)
updated_samples = []
for data in dd.sample_data_iterator(samples):
data = dd.set_combined_counts(data, combined)
if annotated:
data = dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
data = dd.set_combined_fpkm(data, fpkm_combined)
if fpkm_isoform_combined:
data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined)
if express_counts_combined:
data = dd.set_express_counts(data, express_counts_combined['counts'])
data = dd.set_express_tpm(data, express_counts_combined['tpm'])
data = dd.set_express_fpkm(data, express_counts_combined['fpkm'])
data = dd.set_isoform_to_gene(data, express_counts_combined['isoform_to_gene'])
if dexseq_combined:
data = dd.set_dexseq_counts(data, dexseq_combined_file)
updated_samples.append([data])
return updated_samples
def combine_files(samples):
"""
after quantitation, combine the counts/FPKM/TPM/etc into a single table with
all samples
"""
gtf_file = dd.get_gtf_file(samples[0][0], None)
dexseq_gff = dd.get_dexseq_gff(samples[0][0])
# combine featureCount files
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files, ext=".counts")
annotated = count.annotate_combined_count_file(combined, gtf_file)
# combine eXpress files
express_counts_combined = combine_express(samples, combined)
# combine Cufflinks files
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
if fpkm_files:
fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file)
else:
fpkm_combined = None
fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm"
isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples])
if isoform_files:
fpkm_isoform_combined = count.combine_count_files(isoform_files,
fpkm_isoform_combined_file,
".isoform.fpkm")
else:
fpkm_isoform_combined = None
# combine DEXseq files
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data in samples])
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file, ".dexseq")
dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined)
else:
dexseq_combined = None
samples = spikein.combine_spikein(samples)
updated_samples = []
for data in dd.sample_data_iterator(samples):
data = dd.set_combined_counts(data, combined)
if annotated:
data = dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
data = dd.set_combined_fpkm(data, fpkm_combined)
if fpkm_isoform_combined:
data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined)
if express_counts_combined:
data = dd.set_express_counts(data, express_counts_combined['counts'])
data = dd.set_express_tpm(data, express_counts_combined['tpm'])
data = dd.set_express_fpkm(data, express_counts_combined['fpkm'])
data = dd.set_isoform_to_gene(data, express_counts_combined['isoform_to_gene'])
if dexseq_combined:
data = dd.set_dexseq_counts(data, dexseq_combined_file)
updated_samples.append([data])
return updated_samples
def generate_transcript_counts(data):
"""Generate counts per transcript and per exon from an alignment"""
data["count_file"] = featureCounts.count(data)
if dd.get_fusion_mode(data, False):
oncofuse_file = oncofuse.run(data)
if oncofuse_file:
data["oncofuse_file"] = oncofuse.run(data)
if dd.get_dexseq_gff(data, None):
data = dd.set_dexseq_counts(data, dexseq.bcbio_run(data))
return [[data]]
def bcbio_run(data):
out_dir = os.path.join(dd.get_work_dir(data), "dexseq")
safe_makedir(out_dir)
sample_name = dd.get_sample_name(data)
out_file = os.path.join(out_dir, sample_name + ".dexseq")
bam_file = dd.get_work_bam(data)
dexseq_gff = dd.get_dexseq_gff(data)
stranded = dd.get_strandedness(data)
counts = run_count(bam_file, dexseq_gff, stranded, out_file, data)
data = dd.set_dexseq_counts(data, counts)
return data
def bcbio_run(data):
out_dir = os.path.join(dd.get_work_dir(data), "dexseq")
safe_makedir(out_dir)
sample_name = dd.get_sample_name(data)
out_file = os.path.join(out_dir, sample_name + ".dexseq")
bam_file = dd.get_work_bam(data)
dexseq_gff = dd.get_dexseq_gff(data)
stranded = dd.get_strandedness(data)
counts = run_count(bam_file, dexseq_gff, stranded, out_file, data)
data = dd.set_dexseq_counts(data, counts)
return data
def generate_transcript_counts(data):
"""Generate counts per transcript and per exon from an alignment"""
data["count_file"] = featureCounts.count(data)
if dd.get_fusion_mode(data, False):
oncofuse_file = oncofuse.run(data)
if oncofuse_file:
data["oncofuse_file"] = oncofuse.run(data)
if dd.get_dexseq_gff(data, None):
data = dd.set_dexseq_counts(data, dexseq.bcbio_run(data))
# if RSEM was run, stick the transcriptome BAM file into the datadict
if dd.get_aligner(data).lower() == "star" and dd.get_rsem(data):
base, ext = os.path.splitext(dd.get_work_bam(data))
data = dd.set_transcriptome_bam(data, base + ".transcriptome" + ext)
return [[data]]
def generate_transcript_counts(data):
"""Generate counts per transcript and per exon from an alignment"""
data["count_file"] = featureCounts.count(data)
if dd.get_fusion_mode(data, False):
oncofuse_file = oncofuse.run(data)
if oncofuse_file:
data["oncofuse_file"] = oncofuse.run(data)
if dd.get_dexseq_gff(data, None):
data = dd.set_dexseq_counts(data, dexseq.bcbio_run(data))
# if RSEM was run, stick the transcriptome BAM file into the datadict
if dd.get_aligner(data).lower() == "star" and dd.get_rsem(data):
base, ext = os.path.splitext(dd.get_work_bam(data))
data = dd.set_transcriptome_bam(data, base + ".transcriptome" + ext)
return [[data]]
def estimate_expression(samples, run_parallel):
samples = run_parallel("generate_transcript_counts", samples)
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files)
gtf_file = dd.get_gtf_file(samples[0][0], None)
annotated = count.annotate_combined_count_file(combined, gtf_file)
samples = run_parallel("run_cufflinks", samples)
#gene
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file)
#isoform
fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm"
isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples])
fpkm_isoform_combined = count.combine_count_files(isoform_files,
fpkm_isoform_combined_file,
".isoform.fpkm")
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data in samples])
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file, ".dexseq")
else:
dexseq_combined = None
for data in dd.sample_data_iterator(samples):
dd.set_combined_counts(data, combined)
if annotated:
dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
dd.set_combined_fpkm(x[0], fpkm_combined)
if fpkm_isoform_combined:
dd.set_combined_fpkm_isoform(x[0], fpkm_combined)
if dexseq_combined:
dd.set_dexseq_counts(x[0], dexseq_combined_file)
return samples
def estimate_expression(samples, run_parallel):
samples = run_parallel("generate_transcript_counts", samples)
combined = count.combine_count_files(
[x[0]["count_file"] for x in samples if "count_file" in x[0]])
gtf_file = dd.get_gtf_file(samples[0][0], None)
annotated = count.annotate_combined_count_file(combined, gtf_file)
samples = run_parallel("run_cufflinks", samples)
#gene
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
to_combine = [x[0]["fpkm"] for x in samples if "fpkm" in x[0]]
fpkm_combined = count.combine_count_files(to_combine, fpkm_combined_file)
#isoform
fpkm_isoform_combined_file = os.path.splitext(
combined)[0] + ".isoform.fpkm"
to_combine_isoform = [
x[0]["fpkm_isoform"] for x in samples if "fpkm_isoform" in x[0]
]
fpkm_isoform_combined = count.combine_count_files(
to_combine_isoform, fpkm_isoform_combined_file, ".isoform.fpkm")
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = [dd.get_dexseq_counts(data[0]) for data in samples]
to_combine_dexseq = filter(lambda x: x, to_combine_dexseq)
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file,
".dexseq")
else:
dexseq_combined = None
for x in samples:
x[0]["combined_counts"] = combined
if annotated:
x[0]["annotated_combined_counts"] = annotated
if fpkm_combined:
x[0]["combined_fpkm"] = fpkm_combined
if fpkm_isoform_combined:
x[0]["combined_fpkm_isoform"] = fpkm_isoform_combined
if dexseq_combined:
x[0] = dd.set_dexseq_counts(x[0], dexseq_combined_file)
return samples
def estimate_expression(samples, run_parallel):
samples = run_parallel("generate_transcript_counts", samples)
combined = count.combine_count_files([x[0]["count_file"] for x in samples
if "count_file" in x[0]])
gtf_file = dd.get_gtf_file(samples[0][0], None)
annotated = count.annotate_combined_count_file(combined, gtf_file)
samples = run_parallel("run_cufflinks", samples)
#gene
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
to_combine = [x[0]["fpkm"] for x in samples if "fpkm" in x[0]]
fpkm_combined = count.combine_count_files(to_combine, fpkm_combined_file)
#isoform
fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm"
to_combine_isoform = [x[0]["fpkm_isoform"] for x in samples if "fpkm_isoform" in x[0]]
fpkm_isoform_combined = count.combine_count_files(to_combine_isoform,
fpkm_isoform_combined_file,
".isoform.fpkm")
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = [dd.get_dexseq_counts(data[0]) for data in samples]
to_combine_dexseq = filter(lambda x: x, to_combine_dexseq)
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file, ".dexseq")
else:
dexseq_combined = None
for x in samples:
x[0]["combined_counts"] = combined
if annotated:
x[0]["annotated_combined_counts"] = annotated
if fpkm_combined:
x[0]["combined_fpkm"] = fpkm_combined
if fpkm_isoform_combined:
x[0]["combined_fpkm_isoform"] = fpkm_isoform_combined
if dexseq_combined:
x[0] = dd.set_dexseq_counts(x[0], dexseq_combined_file)
return samples
def combine_files(samples):
"""
after quantitation, combine the counts/FPKM/TPM/etc into a single table with
all samples
"""
data = samples[0][0]
# prefer the supplied transcriptome gtf file
gtf_file = dd.get_transcriptome_gtf(data, None)
if not gtf_file:
gtf_file = dd.get_gtf_file(data, None)
dexseq_gff = dd.get_dexseq_gff(data)
# combine featureCount files
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files, ext=".counts")
annotated = count.annotate_combined_count_file(combined, gtf_file)
# add tx2gene file
tx2gene_file = os.path.join(dd.get_work_dir(data), "annotation",
"tx2gene.csv")
if gtf_file:
tx2gene_file = sailfish.create_combined_tx2gene(data)
# combine eXpress files
express_counts_combined = combine_express(samples, combined)
# combine Cufflinks files
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
if fpkm_files:
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_combined = count.combine_count_files(fpkm_files,
fpkm_combined_file)
else:
fpkm_combined = None
isoform_files = filter_missing(
[dd.get_fpkm_isoform(x[0]) for x in samples])
if isoform_files:
fpkm_isoform_combined_file = os.path.splitext(
combined)[0] + ".isoform.fpkm"
fpkm_isoform_combined = count.combine_count_files(
isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm")
else:
fpkm_isoform_combined = None
# combine DEXseq files
to_combine_dexseq = filter_missing(
[dd.get_dexseq_counts(data[0]) for data in samples])
if to_combine_dexseq:
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file,
".dexseq")
dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined)
else:
dexseq_combined = None
samples = spikein.combine_spikein(samples)
updated_samples = []
for data in dd.sample_data_iterator(samples):
if combined:
data = dd.set_combined_counts(data, combined)
if annotated:
data = dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
data = dd.set_combined_fpkm(data, fpkm_combined)
if fpkm_isoform_combined:
data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined)
if express_counts_combined:
data = dd.set_express_counts(data,
express_counts_combined['counts'])
data = dd.set_express_tpm(data, express_counts_combined['tpm'])
data = dd.set_express_fpkm(data, express_counts_combined['fpkm'])
data = dd.set_isoform_to_gene(
data, express_counts_combined['isoform_to_gene'])
if dexseq_combined:
data = dd.set_dexseq_counts(data, dexseq_combined_file)
if gtf_file:
data = dd.set_tx2gene(data, tx2gene_file)
updated_samples.append([data])
return updated_samples
def combine_files(samples):
"""
after quantitation, combine the counts/FPKM/TPM/etc into a single table with
all samples
"""
data = samples[0][0]
# prefer the supplied transcriptome gtf file
gtf_file = dd.get_transcriptome_gtf(data, None)
if not gtf_file:
gtf_file = dd.get_gtf_file(data, None)
dexseq_gff = dd.get_dexseq_gff(data)
# combine featureCount files
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files, ext=".counts")
annotated = count.annotate_combined_count_file(combined, gtf_file)
# add tx2gene file
tx2gene_file = os.path.join(dd.get_work_dir(data), "annotation", "tx2gene.csv")
if gtf_file:
tx2gene_file = sailfish.create_combined_tx2gene(data)
# combine eXpress files
express_counts_combined = combine_express(samples, combined)
# combine Cufflinks files
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
if fpkm_files and combined:
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file)
else:
fpkm_combined = None
isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples])
if isoform_files and combined:
fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm"
fpkm_isoform_combined = count.combine_count_files(isoform_files,
fpkm_isoform_combined_file,
".isoform.fpkm")
else:
fpkm_isoform_combined = None
# combine DEXseq files
to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data
in samples])
if to_combine_dexseq and combined:
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file, ".dexseq")
if dexseq_combined:
dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined)
else:
dexseq_combined = None
samples = spikein.combine_spikein(samples)
updated_samples = []
for data in dd.sample_data_iterator(samples):
if combined:
data = dd.set_combined_counts(data, combined)
if annotated:
data = dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
data = dd.set_combined_fpkm(data, fpkm_combined)
if fpkm_isoform_combined:
data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined)
if express_counts_combined:
data = dd.set_express_counts(data, express_counts_combined['counts'])
data = dd.set_express_tpm(data, express_counts_combined['tpm'])
data = dd.set_express_fpkm(data, express_counts_combined['fpkm'])
data = dd.set_isoform_to_gene(data, express_counts_combined['isoform_to_gene'])
if dexseq_combined:
data = dd.set_dexseq_counts(data, dexseq_combined_file)
if gtf_file:
data = dd.set_tx2gene(data, tx2gene_file)
updated_samples.append([data])
return updated_samples