def run(self, config, config_file, parallel, dirs, samples): with prun.start(parallel, samples, config, dirs, "trimming") as run_parallel: samples = run_parallel("trim_lane", samples) with prun.start( _wprogs(parallel, ["aligner"], {"tophat": 8, "tophat2": 8, "star": 30}), samples, config, dirs, "multicore", multiplier=alignprep.parallel_multiplier(samples), ) as run_parallel: samples = disambiguate.split(samples) samples = run_parallel("process_alignment", samples) samples = disambiguate.resolve(samples, run_parallel) with prun.start( _wprogs(parallel, ["samtools", "gatk", "cufflinks"]), samples, config, dirs, "rnaseqcount" ) as run_parallel: samples = rnaseq.estimate_expression(samples, run_parallel) # samples = rnaseq.detect_fusion(samples, run_parallel) combined = combine_count_files([x[0].get("count_file") for x in samples]) organism = utils.get_in(samples[0][0], ("genome_resources", "aliases", "ensembl"), None) annotated = annotate_combined_count_file(combined, organism) for x in samples: x[0]["combined_counts"] = combined x[0]["annotated_combined_counts"] = annotated with prun.start( _wprogs(parallel, ["picard", "fastqc", "rnaseqc"]), samples, config, dirs, "persample" ) as run_parallel: samples = qcsummary.generate_parallel(samples, run_parallel) return samples
def estimate_expression(samples, run_parallel): samples = run_parallel("generate_transcript_counts", samples) combined = count.combine_count_files([x[0]["count_file"] for x in samples if "count_file" in x[0]]) gtf_file = get_in(samples[0][0], ('genome_resources', 'rnaseq', 'transcripts'), None) annotated = count.annotate_combined_count_file(combined, gtf_file) samples = run_parallel("run_cufflinks", samples) #gene fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" to_combine = [x[0]["fpkm"] for x in samples if "fpkm" in x[0]] fpkm_combined = count.combine_count_files(to_combine, fpkm_combined_file) #isoform fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm" to_combine_isoform = [x[0]["fpkm_isoform"] for x in samples if "fpkm_isoform" in x[0]] fpkm_isoform_combined = count.combine_count_files(to_combine_isoform, fpkm_isoform_combined_file, ".isoform.fpkm") for x in samples: x[0]["combined_counts"] = combined if annotated: x[0]["annotated_combined_counts"] = annotated if fpkm_combined: x[0]["combined_fpkm"] = fpkm_combined if fpkm_isoform_combined: x[0]["combined_fpkm_isoform"] = fpkm_isoform_combined return samples
def run(self, config, config_file, parallel, dirs, samples): with prun.start(_wres(parallel, ["picard"]), samples, config, dirs, "trimming") as run_parallel: samples = run_parallel("process_lane", samples) samples = run_parallel("trim_lane", samples) with prun.start(_wres(parallel, ["aligner"], ensure_mem={"tophat": 8, "tophat2": 8, "star": 30}), samples, config, dirs, "multicore", multiplier=alignprep.parallel_multiplier(samples)) as run_parallel: samples = disambiguate.split(samples) samples = run_parallel("process_alignment", samples) samples = disambiguate.resolve(samples, run_parallel) with prun.start(_wres(parallel, ["samtools", "cufflinks"]), samples, config, dirs, "rnaseqcount") as run_parallel: samples = rnaseq.estimate_expression(samples, run_parallel) #samples = rnaseq.detect_fusion(samples, run_parallel) combined = combine_count_files([x[0].get("count_file") for x in samples]) gtf_file = utils.get_in(samples[0][0], ('genome_resources', 'rnaseq', 'transcripts'), None) annotated = annotate_combined_count_file(combined, gtf_file) for x in samples: x[0]["combined_counts"] = combined if annotated: x[0]["annotated_combined_counts"] = annotated with prun.start(_wres(parallel, ["picard", "fastqc", "rnaseqc"]), samples, config, dirs, "persample") as run_parallel: samples = qcsummary.generate_parallel(samples, run_parallel) return samples
def combine_files(samples): """ after quantitation, combine the counts/FPKM/TPM/etc into a single table with all samples """ gtf_file = dd.get_gtf_file(samples[0][0], None) dexseq_gff = dd.get_dexseq_gff(samples[0][0]) # combine featureCount files count_files = filter_missing([dd.get_count_file(x[0]) for x in samples]) combined = count.combine_count_files(count_files, ext=".counts") annotated = count.annotate_combined_count_file(combined, gtf_file) # combine eXpress files express_counts_combined = combine_express(samples, combined) # combine Cufflinks files fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples]) if fpkm_files: fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file) else: fpkm_combined = None fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm" isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples]) if isoform_files: fpkm_isoform_combined = count.combine_count_files(isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm") else: fpkm_isoform_combined = None # combine DEXseq files dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data in samples]) if to_combine_dexseq: dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined) else: dexseq_combined = None samples = spikein.combine_spikein(samples) updated_samples = [] for data in dd.sample_data_iterator(samples): data = dd.set_combined_counts(data, combined) if annotated: data = dd.set_annotated_combined_counts(data, annotated) if fpkm_combined: data = dd.set_combined_fpkm(data, fpkm_combined) if fpkm_isoform_combined: data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined) if express_counts_combined: data = dd.set_express_counts(data, express_counts_combined['counts']) data = dd.set_express_tpm(data, express_counts_combined['tpm']) data = dd.set_express_fpkm(data, express_counts_combined['fpkm']) data = dd.set_isoform_to_gene(data, express_counts_combined['isoform_to_gene']) if dexseq_combined: data = dd.set_dexseq_counts(data, dexseq_combined_file) updated_samples.append([data]) return updated_samples
def estimate_expression(samples, run_parallel): samples = run_parallel("generate_transcript_counts", samples) count_files = filter_missing([dd.get_count_file(x[0]) for x in samples]) combined = count.combine_count_files(count_files) gtf_file = dd.get_gtf_file(samples[0][0], None) annotated = count.annotate_combined_count_file(combined, gtf_file) samples = run_parallel("run_express", samples) express_counts_combined = combine_express(samples, combined) samples = run_parallel("run_cufflinks", samples) #gene fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples]) fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file) #isoform fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm" isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples]) fpkm_isoform_combined = count.combine_count_files(isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm") dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data in samples]) if to_combine_dexseq: dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") else: dexseq_combined = None updated_samples = [] for data in dd.sample_data_iterator(samples): data = dd.set_combined_counts(data, combined) if annotated: data = dd.set_annotated_combined_counts(data, annotated) if fpkm_combined: data = dd.set_combined_fpkm(data, fpkm_combined) if fpkm_isoform_combined: data = dd.set_combined_fpkm_isoform(data, fpkm_combined) if express_counts_combined: data = dd.set_express_counts(data, express_counts_combined['counts']) data = dd.set_express_tpm(data, express_counts_combined['tpm']) data = dd.set_express_fpkm(data, express_counts_combined['fpkm']) if dexseq_combined: data = dd.set_dexseq_counts(data, dexseq_combined_file) updated_samples.append([data]) return updated_samples
def run(self, config, config_file, parallel, dirs, samples): with prun.start(_wres(parallel, ["picard", "AlienTrimmer"]), samples, config, dirs, "trimming") as run_parallel: with profile.report("adapter trimming", dirs): samples = run_parallel("process_lane", samples) samples = run_parallel("trim_lane", samples) with prun.start(_wres(parallel, ["aligner"], ensure_mem={ "tophat": 8, "tophat2": 8, "star": 30 }), samples, config, dirs, "multicore", multiplier=alignprep.parallel_multiplier( samples)) as run_parallel: with profile.report("alignment", dirs): samples = disambiguate.split(samples) samples = run_parallel("process_alignment", samples) with prun.start(_wres(parallel, ["samtools", "cufflinks"]), samples, config, dirs, "rnaseqcount") as run_parallel: with profile.report("disambiguation", dirs): samples = disambiguate.resolve(samples, run_parallel) with profile.report("estimate expression", dirs): samples = rnaseq.estimate_expression(samples, run_parallel) combined = combine_count_files( [x[0].get("count_file") for x in samples]) gtf_file = utils.get_in(samples[0][0], ('genome_resources', 'rnaseq', 'transcripts'), None) annotated = annotate_combined_count_file(combined, gtf_file) for x in samples: x[0]["combined_counts"] = combined if annotated: x[0]["annotated_combined_counts"] = annotated with prun.start(_wres(parallel, ["picard", "fastqc", "rnaseqc"]), samples, config, dirs, "persample") as run_parallel: with profile.report("quality control", dirs): samples = qcsummary.generate_parallel(samples, run_parallel) logger.info("Timing: finished") return samples
def run(self, config, config_file, run_parallel, parallel, dirs, lane_items): lane_items = run_parallel("trim_lane", lane_items) samples = disambiguate.split(lane_items) samples = run_parallel("process_alignment", samples) samples = disambiguate.resolve(samples, run_parallel) samples = rnaseq.estimate_expression(samples, run_parallel) #samples = rnaseq.detect_fusion(samples, run_parallel) combined = combine_count_files([x[0].get("count_file") for x in samples]) organism = utils.get_in(samples[0][0], ('genome_resources', 'aliases', 'ensembl'), None) annotated = annotate_combined_count_file(combined, organism) for x in samples: x[0]["combined_counts"] = combined x[0]["annotated_combined_counts"] = annotated samples = qcsummary.generate_parallel(samples, run_parallel) #run_parallel("generate_bigwig", samples, {"programs": ["ucsc_bigwig"]}) return samples
def estimate_expression(samples, run_parallel): samples = run_parallel("generate_transcript_counts", samples) combined = count.combine_count_files( [x[0]["count_file"] for x in samples if "count_file" in x[0]]) gtf_file = dd.get_gtf_file(samples[0][0], None) annotated = count.annotate_combined_count_file(combined, gtf_file) samples = run_parallel("run_cufflinks", samples) #gene fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" to_combine = [x[0]["fpkm"] for x in samples if "fpkm" in x[0]] fpkm_combined = count.combine_count_files(to_combine, fpkm_combined_file) #isoform fpkm_isoform_combined_file = os.path.splitext( combined)[0] + ".isoform.fpkm" to_combine_isoform = [ x[0]["fpkm_isoform"] for x in samples if "fpkm_isoform" in x[0] ] fpkm_isoform_combined = count.combine_count_files( to_combine_isoform, fpkm_isoform_combined_file, ".isoform.fpkm") dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" to_combine_dexseq = [dd.get_dexseq_counts(data[0]) for data in samples] to_combine_dexseq = filter(lambda x: x, to_combine_dexseq) if to_combine_dexseq: dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") else: dexseq_combined = None for x in samples: x[0]["combined_counts"] = combined if annotated: x[0]["annotated_combined_counts"] = annotated if fpkm_combined: x[0]["combined_fpkm"] = fpkm_combined if fpkm_isoform_combined: x[0]["combined_fpkm_isoform"] = fpkm_isoform_combined if dexseq_combined: x[0] = dd.set_dexseq_counts(x[0], dexseq_combined_file) return samples
def estimate_expression(samples, run_parallel): samples = run_parallel("generate_transcript_counts", samples) combined = count.combine_count_files([x[0]["count_file"] for x in samples if "count_file" in x[0]]) gtf_file = dd.get_gtf_file(samples[0][0], None) annotated = count.annotate_combined_count_file(combined, gtf_file) samples = run_parallel("run_cufflinks", samples) #gene fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" to_combine = [x[0]["fpkm"] for x in samples if "fpkm" in x[0]] fpkm_combined = count.combine_count_files(to_combine, fpkm_combined_file) #isoform fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm" to_combine_isoform = [x[0]["fpkm_isoform"] for x in samples if "fpkm_isoform" in x[0]] fpkm_isoform_combined = count.combine_count_files(to_combine_isoform, fpkm_isoform_combined_file, ".isoform.fpkm") dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" to_combine_dexseq = [dd.get_dexseq_counts(data[0]) for data in samples] to_combine_dexseq = filter(lambda x: x, to_combine_dexseq) if to_combine_dexseq: dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") else: dexseq_combined = None for x in samples: x[0]["combined_counts"] = combined if annotated: x[0]["annotated_combined_counts"] = annotated if fpkm_combined: x[0]["combined_fpkm"] = fpkm_combined if fpkm_isoform_combined: x[0]["combined_fpkm_isoform"] = fpkm_isoform_combined if dexseq_combined: x[0] = dd.set_dexseq_counts(x[0], dexseq_combined_file) return samples
def combine_files(samples): """ after quantitation, combine the counts/FPKM/TPM/etc into a single table with all samples """ data = samples[0][0] # prefer the supplied transcriptome gtf file gtf_file = dd.get_transcriptome_gtf(data, None) if not gtf_file: gtf_file = dd.get_gtf_file(data, None) dexseq_gff = dd.get_dexseq_gff(data) # combine featureCount files count_files = filter_missing([dd.get_count_file(x[0]) for x in samples]) combined = count.combine_count_files(count_files, ext=".counts") annotated = count.annotate_combined_count_file(combined, gtf_file) # add tx2gene file tx2gene_file = os.path.join(dd.get_work_dir(data), "annotation", "tx2gene.csv") if gtf_file: tx2gene_file = sailfish.create_combined_tx2gene(data) # combine eXpress files express_counts_combined = combine_express(samples, combined) # combine Cufflinks files fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples]) if fpkm_files: fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file) else: fpkm_combined = None isoform_files = filter_missing( [dd.get_fpkm_isoform(x[0]) for x in samples]) if isoform_files: fpkm_isoform_combined_file = os.path.splitext( combined)[0] + ".isoform.fpkm" fpkm_isoform_combined = count.combine_count_files( isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm") else: fpkm_isoform_combined = None # combine DEXseq files to_combine_dexseq = filter_missing( [dd.get_dexseq_counts(data[0]) for data in samples]) if to_combine_dexseq: dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined) else: dexseq_combined = None samples = spikein.combine_spikein(samples) updated_samples = [] for data in dd.sample_data_iterator(samples): if combined: data = dd.set_combined_counts(data, combined) if annotated: data = dd.set_annotated_combined_counts(data, annotated) if fpkm_combined: data = dd.set_combined_fpkm(data, fpkm_combined) if fpkm_isoform_combined: data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined) if express_counts_combined: data = dd.set_express_counts(data, express_counts_combined['counts']) data = dd.set_express_tpm(data, express_counts_combined['tpm']) data = dd.set_express_fpkm(data, express_counts_combined['fpkm']) data = dd.set_isoform_to_gene( data, express_counts_combined['isoform_to_gene']) if dexseq_combined: data = dd.set_dexseq_counts(data, dexseq_combined_file) if gtf_file: data = dd.set_tx2gene(data, tx2gene_file) updated_samples.append([data]) return updated_samples
def combine_files(samples): """ after quantitation, combine the counts/FPKM/TPM/etc into a single table with all samples """ gtf_file = dd.get_gtf_file(samples[0][0], None) dexseq_gff = dd.get_dexseq_gff(samples[0][0]) # combine featureCount files count_files = filter_missing([dd.get_count_file(x[0]) for x in samples]) combined = count.combine_count_files(count_files, ext=".counts") annotated = count.annotate_combined_count_file(combined, gtf_file) # combine eXpress files express_counts_combined = combine_express(samples, combined) # combine Cufflinks files fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples]) if fpkm_files: fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file) else: fpkm_combined = None fpkm_isoform_combined_file = os.path.splitext( combined)[0] + ".isoform.fpkm" isoform_files = filter_missing( [dd.get_fpkm_isoform(x[0]) for x in samples]) if isoform_files: fpkm_isoform_combined = count.combine_count_files( isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm") else: fpkm_isoform_combined = None # combine DEXseq files dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" to_combine_dexseq = filter_missing( [dd.get_dexseq_counts(data[0]) for data in samples]) if to_combine_dexseq: dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined) else: dexseq_combined = None samples = spikein.combine_spikein(samples) updated_samples = [] for data in dd.sample_data_iterator(samples): data = dd.set_combined_counts(data, combined) if annotated: data = dd.set_annotated_combined_counts(data, annotated) if fpkm_combined: data = dd.set_combined_fpkm(data, fpkm_combined) if fpkm_isoform_combined: data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined) if express_counts_combined: data = dd.set_express_counts(data, express_counts_combined['counts']) data = dd.set_express_tpm(data, express_counts_combined['tpm']) data = dd.set_express_fpkm(data, express_counts_combined['fpkm']) data = dd.set_isoform_to_gene( data, express_counts_combined['isoform_to_gene']) if dexseq_combined: data = dd.set_dexseq_counts(data, dexseq_combined_file) updated_samples.append([data]) return updated_samples
def combine_files(samples): """ after quantitation, combine the counts/FPKM/TPM/etc into a single table with all samples """ data = samples[0][0] # prefer the supplied transcriptome gtf file gtf_file = dd.get_transcriptome_gtf(data, None) if not gtf_file: gtf_file = dd.get_gtf_file(data, None) dexseq_gff = dd.get_dexseq_gff(data) # combine featureCount files count_files = filter_missing([dd.get_count_file(x[0]) for x in samples]) combined = count.combine_count_files(count_files, ext=".counts") annotated = count.annotate_combined_count_file(combined, gtf_file) # add tx2gene file tx2gene_file = os.path.join(dd.get_work_dir(data), "annotation", "tx2gene.csv") if gtf_file: tx2gene_file = sailfish.create_combined_tx2gene(data) # combine eXpress files express_counts_combined = combine_express(samples, combined) # combine Cufflinks files fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples]) if fpkm_files and combined: fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm" fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file) else: fpkm_combined = None isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples]) if isoform_files and combined: fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm" fpkm_isoform_combined = count.combine_count_files(isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm") else: fpkm_isoform_combined = None # combine DEXseq files to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data in samples]) if to_combine_dexseq and combined: dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq" dexseq_combined = count.combine_count_files(to_combine_dexseq, dexseq_combined_file, ".dexseq") if dexseq_combined: dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined) else: dexseq_combined = None samples = spikein.combine_spikein(samples) updated_samples = [] for data in dd.sample_data_iterator(samples): if combined: data = dd.set_combined_counts(data, combined) if annotated: data = dd.set_annotated_combined_counts(data, annotated) if fpkm_combined: data = dd.set_combined_fpkm(data, fpkm_combined) if fpkm_isoform_combined: data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined) if express_counts_combined: data = dd.set_express_counts(data, express_counts_combined['counts']) data = dd.set_express_tpm(data, express_counts_combined['tpm']) data = dd.set_express_fpkm(data, express_counts_combined['fpkm']) data = dd.set_isoform_to_gene(data, express_counts_combined['isoform_to_gene']) if dexseq_combined: data = dd.set_dexseq_counts(data, dexseq_combined_file) if gtf_file: data = dd.set_tx2gene(data, tx2gene_file) updated_samples.append([data]) return updated_samples