def combine_sailfish(samples):
work_dir = dd.get_in_samples(samples, dd.get_work_dir)
sailfish_dir = os.path.join(work_dir, "sailfish")
gtf_file = dd.get_in_samples(samples, dd.get_gtf_file)
dont_combine, to_combine = partition(dd.get_sailfish,
dd.sample_data_iterator(samples), True)
if not to_combine:
return samples
tidy_file = os.path.join(sailfish_dir, "combined.sf")
transcript_tpm_file = os.path.join(sailfish_dir, "combined.isoform.sf.tpm")
gene_tpm_file = os.path.join(sailfish_dir, "combined.gene.sf.tpm")
tx2gene = os.path.join(sailfish_dir, "tx2gene.csv")
if not all([file_exists(x) for x in [gene_tpm_file, tidy_file,
transcript_tpm_file, tx2gene]]):
logger.info("Combining count files into %s." % tidy_file)
df = pd.DataFrame()
for data in to_combine:
sailfish_file = dd.get_sailfish(data)
samplename = dd.get_sample_name(data)
new_df = _sailfish_expression_parser(sailfish_file, samplename)
if df.empty:
df = new_df
else:
df = rbind([df, new_df])
df["id"] = df.index
# some versions of the transcript annotations can have duplicated entries
df = df.drop_duplicates(["id", "sample"])
with file_transaction(tidy_file) as tx_out_file:
df.to_csv(tx_out_file, sep="\t", index_label="name")
with file_transaction(transcript_tpm_file) as tx_out_file:
df.pivot("id", "sample", "tpm").to_csv(tx_out_file, sep="\t")
with file_transaction(gene_tpm_file) as tx_out_file:
pivot = df.pivot("id", "sample", "tpm")
tdf = pd.DataFrame.from_dict(gtf.transcript_to_gene(gtf_file),
orient="index")
tdf.columns = ["gene_id"]
pivot = pivot.join(tdf)
pivot = pivot.groupby("gene_id").agg(np.sum)
pivot.to_csv(tx_out_file, sep="\t")
tx2gene = gtf.tx2genefile(gtf_file, tx2gene)
logger.info("Finished combining count files into %s." % tidy_file)
updated_samples = []
for data in dd.sample_data_iterator(samples):
data = dd.set_sailfish_tidy(data, tidy_file)
data = dd.set_sailfish_transcript_tpm(data, transcript_tpm_file)
data = dd.set_sailfish_gene_tpm(data, gene_tpm_file)
data = dd.set_tx2gene(data, tx2gene)
updated_samples.append([data])
return updated_samples
def combine_files(samples):
"""
after quantitation, combine the counts/FPKM/TPM/etc into a single table with
all samples
"""
data = samples[0][0]
gtf_file = dd.get_gtf_file(data, None)
dexseq_gff = dd.get_dexseq_gff(data)
# combine featureCount files
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files, ext=".counts")
annotated = count.annotate_combined_count_file(combined, gtf_file)
# add tx2gene file
tx2gene_file = os.path.join(dd.get_work_dir(data), "annotation",
"tx2gene.csv")
if gtf_file:
tx2gene_file = tx2genefile(gtf_file, tx2gene_file)
# combine eXpress files
express_counts_combined = combine_express(samples, combined)
# combine Cufflinks files
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
if fpkm_files:
fpkm_combined = count.combine_count_files(fpkm_files,
fpkm_combined_file)
else:
fpkm_combined = None
fpkm_isoform_combined_file = os.path.splitext(
combined)[0] + ".isoform.fpkm"
isoform_files = filter_missing(
[dd.get_fpkm_isoform(x[0]) for x in samples])
if isoform_files:
fpkm_isoform_combined = count.combine_count_files(
isoform_files, fpkm_isoform_combined_file, ".isoform.fpkm")
else:
fpkm_isoform_combined = None
# combine DEXseq files
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = filter_missing(
[dd.get_dexseq_counts(data[0]) for data in samples])
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file,
".dexseq")
dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined)
else:
dexseq_combined = None
samples = spikein.combine_spikein(samples)
updated_samples = []
for data in dd.sample_data_iterator(samples):
data = dd.set_combined_counts(data, combined)
if annotated:
data = dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
data = dd.set_combined_fpkm(data, fpkm_combined)
if fpkm_isoform_combined:
data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined)
if express_counts_combined:
data = dd.set_express_counts(data,
express_counts_combined['counts'])
data = dd.set_express_tpm(data, express_counts_combined['tpm'])
data = dd.set_express_fpkm(data, express_counts_combined['fpkm'])
data = dd.set_isoform_to_gene(
data, express_counts_combined['isoform_to_gene'])
if dexseq_combined:
data = dd.set_dexseq_counts(data, dexseq_combined_file)
if gtf_file:
data = dd.set_tx2gene(data, tx2gene_file)
updated_samples.append([data])
return updated_samples
def combine_files(samples):
"""
after quantitation, combine the counts/FPKM/TPM/etc into a single table with
all samples
"""
data = samples[0][0]
gtf_file = dd.get_gtf_file(data, None)
dexseq_gff = dd.get_dexseq_gff(data)
# combine featureCount files
count_files = filter_missing([dd.get_count_file(x[0]) for x in samples])
combined = count.combine_count_files(count_files, ext=".counts")
annotated = count.annotate_combined_count_file(combined, gtf_file)
# add tx2gene file
tx2gene_file = os.path.join(dd.get_work_dir(data), "annotation", "tx2gene.csv")
if gtf_file:
tx2gene_file = tx2genefile(gtf_file, tx2gene_file)
# combine eXpress files
express_counts_combined = combine_express(samples, combined)
# combine Cufflinks files
fpkm_combined_file = os.path.splitext(combined)[0] + ".fpkm"
fpkm_files = filter_missing([dd.get_fpkm(x[0]) for x in samples])
if fpkm_files:
fpkm_combined = count.combine_count_files(fpkm_files, fpkm_combined_file)
else:
fpkm_combined = None
fpkm_isoform_combined_file = os.path.splitext(combined)[0] + ".isoform.fpkm"
isoform_files = filter_missing([dd.get_fpkm_isoform(x[0]) for x in samples])
if isoform_files:
fpkm_isoform_combined = count.combine_count_files(isoform_files,
fpkm_isoform_combined_file,
".isoform.fpkm")
else:
fpkm_isoform_combined = None
# combine DEXseq files
dexseq_combined_file = os.path.splitext(combined)[0] + ".dexseq"
to_combine_dexseq = filter_missing([dd.get_dexseq_counts(data[0]) for data
in samples])
if to_combine_dexseq:
dexseq_combined = count.combine_count_files(to_combine_dexseq,
dexseq_combined_file, ".dexseq")
dexseq.create_dexseq_annotation(dexseq_gff, dexseq_combined)
else:
dexseq_combined = None
samples = spikein.combine_spikein(samples)
updated_samples = []
for data in dd.sample_data_iterator(samples):
data = dd.set_combined_counts(data, combined)
if annotated:
data = dd.set_annotated_combined_counts(data, annotated)
if fpkm_combined:
data = dd.set_combined_fpkm(data, fpkm_combined)
if fpkm_isoform_combined:
data = dd.set_combined_fpkm_isoform(data, fpkm_isoform_combined)
if express_counts_combined:
data = dd.set_express_counts(data, express_counts_combined['counts'])
data = dd.set_express_tpm(data, express_counts_combined['tpm'])
data = dd.set_express_fpkm(data, express_counts_combined['fpkm'])
data = dd.set_isoform_to_gene(data, express_counts_combined['isoform_to_gene'])
if dexseq_combined:
data = dd.set_dexseq_counts(data, dexseq_combined_file)
if gtf_file:
data = dd.set_tx2gene(data, tx2gene_file)
updated_samples.append([data])
return updated_samples
