def makeRelErrorPlot(truthCol, predCol, df, outBase, method, measure):
plt.cla()
plt.clf()
if measure == "tpm":
outFile = "{}_tpm_rel_error.pdf".format(outBase)
elif measure == "num_reads":
outFile = "{}_num_reads_rel_error.pdf".format(outBase)
# Get the relative difference numbers
reV = AnalysisUtils.relError(truthCol, predCol, df)
_ = plt.hist(np.clip(reV, -20.0, 20.0), range=(-20, 20), bins=100, histtype="stepfilled")
mdText = "mean rel error = {0:.2f}\nmean |rel error| = {1:.2f}\n# data points = {2}".format(
reV.mean(), reV.abs().mean(), len(reV))
plt.figtext(0.15, 0.85, mdText)
ax = plt.axes()
ax.set_ylabel("frequency")
ax.set_xlabel("relative difference")
# Get rid of axis spines
sns.despine()
plt.savefig(outFile)
def makeRelDiffPlot(truthCol, predCol, df, outBase, method, measure):
plt.cla()
plt.clf()
if measure == "tpm":
outFile = "{}_tpm_rel_diff.pdf".format(outBase)
elif measure == "num_reads":
outFile = "{}_num_reads_rel_diff.pdf".format(outBase)
# Get the relative difference numbers
rdV, nzV = AnalysisUtils.relDiff(truthCol, predCol, df)
try:
_ = plt.hist(rdV["relDiff"], bins=100, histtype="stepfilled")
mdText = "mean rel diff = {0:.2f}\nmean |rel diff| = {1:.2f}".format(
rdV["relDiff"].mean(), rdV["relDiff"].abs().mean())
plt.figtext(0.15, 0.85, mdText)
ax = plt.axes()
ax.set_ylabel("frequency")
ax.set_xlabel("relative difference")
# Get rid of axis spines
sns.despine()
plt.savefig(outFile)
except:
f = open(outFile, 'w')
f.close()
print("Error making relative difference plot {}".format(outFile))
def proportionalityCorrelationPlots(D, measure, methods, simIDs, outdir, plotname):
sns.set_style('white')
sns.set_context('poster')
plt.clf()
plt.cla()
md = {}
for m in methods:
md[m] = []
for i in simIDs:
pc = AnalysisUtils.proportionalityCorrelation(
"{}_truth{}".format(measure, str(i)),
"{}_{}{}".format(measure, m, str(i)), D)
md[m].append(pc)
ax = None
for mn, mv in md.items():
weights = np.ones_like(mv)/len(mv)
ax = sns.distplot(mv, hist_kws={"histtype": "stepfilled"}, kde=False, rug=True, label=mn, ax=ax, norm_hist=False)
sns.despine()
ax = plt.axes()
ax.set_xlabel('proportionality correlation')
ax.set_ylabel('frequency')
plt.legend()
plt.savefig('{}/{}.pdf'.format(outdir, plotname))
def makePlots(truthFile, predFile, method, outBase, measure):
if measure == "tpm":
trueColName = "TPM_truth"
predColName = "TPM_{}".format(method)
elif measure == "num_reads":
trueColName = "NumReads_truth"
predColName = "NumReads_{}".format(method)
# Load the data; first the predicitons
p = None
if method == "salmon":
p = ParsingUtils.readSalmon(predFile, '_{}'.format(method))
elif method == "kallisto":
p = ParsingUtils.readKallisto(predFile, '_{}'.format(method))
elif method == "express":
p = ParsingUtils.readExpress(predFile, '_{}'.format(method))
# Now the ground truth
g = ParsingUtils.readProFile(truthFile, '_truth')
# Convert to TPM
g["TPM_truth"] = 1000000.0 * (g["ExpFrac_truth"] / g["ExpFrac_truth"].sum())
# Flux sim thinks paired-end = 2 reads . . . sigh
g["NumReads_truth"] = g["SeqNum_truth"] * 0.5
# Filter out low TPM
AnalysisUtils.filterValues("TPM_truth", g, 0.01)
AnalysisUtils.filterValues("TPM_{}".format(method), p, 0.01)
AnalysisUtils.filterValues("NumReads_truth", g, 1.0)
AnalysisUtils.filterValues("NumReads_{}".format(method), p, 1.0)
# merge dataframes
m = g.join(p)
setPlotProperties()
makeCorrPlot(trueColName, predColName, m, outBase, method, measure)
makeRelDiffPlot(trueColName, predColName, m, outBase, method, measure)
makeRelErrorPlot(trueColName, predColName, m, outBase, method, measure)
def rsemRelDiffPlots(D, measure, methods, simIDs, outdir, plotname):
sns.set_style('white')
sns.set_context('poster')
plt.clf()
plt.cla()
md = {}
for m in methods:
md[m] = []
for i in simIDs:
rd, _ = AnalysisUtils.relDiff("{}_truth{}".format(measure, str(i)),
"{}_{}{}".format(measure, m, str(i)), D)
md[m].append(rd['relDiff'].abs().mean())
ax = None
for mn, mv in md.items():
weights = np.ones_like(mv)/len(mv)
ax = sns.distplot(mv, hist_kws={"histtype": "stepfilled"},kde=False, rug=True, label=mn, ax=ax, norm_hist=False)
sns.despine()
ax = plt.axes()
ax.set_xlabel('mean absolute relative difference')
ax.set_ylabel('frequency')
plt.legend()
plt.savefig('{}/{}.pdf'.format(outdir, plotname))
def makeTable(methodDict, outpath, outfile, measure, annotPath):
import pandas as pd
import seaborn as sns
import ParsingUtils
import AnalysisUtils
dframes = []
for k, v in methodDict.items():
if k.upper().startswith('SALMON'):
d = ParsingUtils.readSalmon(v, '_{}'.format(k))
elif k.upper().startswith('KALLISTO'):
d = ParsingUtils.readKallisto(v, '_{}'.format(k))
elif k.upper().startswith('EXPRESS'):
d = ParsingUtils.readExpress(v, '_{}'.format(k))
elif k.upper() == 'SAILFISH':
d = ParsingUtils.readSailfish(v, '_{}'.format(k))
elif k.upper() == 'SAILFISH (QUASI)':
d = ParsingUtils.readSalmon(v, '_{}'.format(k))
elif k.upper().startswith('TRUTH'):
suffix = '_{}'.format(k)
d = ParsingUtils.readProFile(v, suffix)
d["TPM{}".format(suffix)] = 1000000.0 * (d["ExpFrac{}".format(suffix)] / d["ExpFrac{}".format(suffix)].sum())
# Flux sim thinks paired-end = 2 reads . . . sinh
d["NumReads{}".format(suffix)] = d["SeqNum{}".format(suffix)] * 0.5
# Add this dataframe to the list
dframes.append(d)
M = dframes[0].join(dframes[1:])
# Filter eXpress results
minVal = np.inf
for mn in set(methodDict.keys()) - set(["Truth", "eXpress"]):
newMin = M.loc[M["{}_{}".format(measure, mn)]>0, "{}_{}".format(measure,mn)].min()
minVal = min(minVal, newMin)
print("filtering eXpress results < {} {}".format(minVal, measure))
AnalysisUtils.filterValues("{}_{}".format(measure, "eXpress"), M, minVal)
org = outfile.split('/')[-1].split('_')[0]
print("org = {}".format(org))
if org == 'human':
plotStratifiedDiffs(M, methodDict, annotPath, outpath, measure)
mrdName = 'abs. mean rel. diff.'
corrName = 'Spearman corr.'
propName = 'Proportionality corr.'
tpefName = 'TP error fraction'
tpMedErrorName = 'TP median per. error'
res = pd.DataFrame(data={ m : {tpMedErrorName : np.nan, tpefName : np.nan, mrdName : np.nan, corrName : np.nan, propName : np.nan} for m in (methodDict.keys() - set('Truth'))})
import scipy as sp
import scipy.stats
for k in methodDict:
if k.upper() != "TRUTH":
c = sp.stats.spearmanr(M["{}_Truth".format(measure)], M["{}_{}".format(measure, k)])[0]
res[k][corrName] = c
mrd, _ = AnalysisUtils.relDiff("{}_Truth".format(measure), "{}_{}".format(measure, k), M)
res[k][mrdName] = mrd["relDiff"].abs().mean()
pc = AnalysisUtils.proportionalityCorrelation("{}_Truth".format(measure), "{}_{}".format(measure, k), M)
res[k][propName] = pc
tpind = M[M["{}_Truth".format(measure)] >= 1]
y = tpind["{}_{}".format(measure, k)]
x = tpind["{}_Truth".format(measure)]
ef = 10.0
re = (y - x) / x
are = 100.0 * (y - x).abs() / x
tpef = len(are[are > ef]) / float(len(are))
res[k][tpefName] = tpef
res[k][tpMedErrorName] = re.median()
res.drop('Truth', axis=1, inplace=True)
print(res)
res.to_csv(outfile+".csv")
with open(outfile, 'w') as ofile:
ofile.write(res.to_latex(float_format=lambda x: "{0:.2f}".format(x)))
print("wrote {}".format(outpath))
def plotStratifiedDiffs(M, methodDict, annotPath, outpath, measure):
import pickle
import seaborn as sns
tgmap = {}
with open('{}/tgmap.txt'.format(annotPath),'r') as ifile:
for l in ifile:
toks = l.split()
tgmap[toks[0]] = toks[1]
genes = pd.DataFrame([(k,v) for k,v in tgmap.items()], columns=['Name', 'Gene'])
genes.set_index('Name', inplace=True)
M = M.join(genes)
relDiffDict = {}
for mn, mf in methodDict.items():
rdiffs, _ = AnalysisUtils.relDiff('{}_Truth'.format(measure), '{}_{}'.format(measure, mn), M)
M['RelDiff_{}'.format(mn)] = rdiffs
mByGenes = M.groupby('Gene')
groups = dict(list(mByGenes))
ntmap = {}
methods = ["Sailfish", "Salmon", "eXpress"]
def retainedMethod(mn):
return mn in methods
sns.set_palette(sns.color_palette([methodcolor[m] for m in methods]))
numExpressedGenes = 0
for i,(k,group) in enumerate(groups.items()):
if i % 10000 == 0:
print("processing group {} of {}".format(i, len(groups)))
## remove 0s?
if group['NumReads_Truth'].sum() == 0:
continue
numExpressedGenes += 1
numTran = len(group)
if numTran not in ntmap:
ntmap[numTran] = {}
for mn, mf in methodDict.items():
if retainedMethod(mn):
#if not mn.upper().endswith('TRUTH'):
ntmap[numTran][mn] = []
for mn, mf in methodDict.items():
if retainedMethod(mn):
#if not mn.upper().endswith('TRUTH'):
ntmap[numTran][mn].append(group['RelDiff_{}'.format(mn)].abs().mean())
print("There were {} expressed genes (genes with at least 1 expressed isoform)".format(numExpressedGenes))
nt = [set([])]
totInClass = 0
classSize = 1000
for k in sorted(ntmap.keys()):
if k > 1:
vd = ntmap[k]
#num = len(vd['Kallisto'])
num = len(vd['Sailfish'])
if len(nt[-1]) == 0 or totInClass <= classSize:
nt[-1].add(k)
totInClass += num
if totInClass > classSize:
totInClass = 0
nt.append(set([]))
print(nt)
ntClasses = {}
for c in nt:
# make a name for the class
minVal = min(list(c))
maxVal = max(list(c))
if minVal == maxVal:
ntClasses[minVal] = str(minVal)
else:
name = str(minVal)+"-"+str(maxVal)
for x in c:
ntClasses[x] = name
from pprint import pprint
pprint(ntClasses)
data = []
for k, v in ntmap.items():
if k > 1:
for mn, vals in v.items():
for x in vals:
cname = ntClasses[k]
data.append((cname, mn, x))
ntxp = '# txp. per gene'
D = pd.DataFrame(data, columns=[ntxp, 'Method', 'MARDs'])
sns.set_style('white')
#sns.set_palette('pastel', desat=0.7)
plt.clf()
plt.cla()
plt.figure(figsize=(12,8))
cnames = sorted(list(set([v for k,v in ntClasses.items()])))
print("Classes = ")
pprint(cnames)
x_order = sorted(cnames, key=lambda x : int(x.split('-')[0]))
import matplotlib
matplotlib.rc("lines", markersize=0, markeredgewidth=0)
g = sns.factorplot(ntxp, "MARDs", "Method", D, kind='box', markers='', x_order=x_order)
#g = sns.facetgrid(ntxp, "MARDs", "Method", D)
g.set_xticklabels(rotation=30)
#g.despine(offset=10, trim=True)
#plt.tight_layout()
plt.gcf().subplots_adjust(bottom=0.15)
plt.savefig('{}/GeneStratRD.pdf'.format(outpath))
def rsemComparisonPlots(topdir, measure, simIDs, outdir):
import os
kallistoQuant = "abundance.txt"
salmonQuant = "quant.sf"
sailfishQuant = "quant.sf/quant.sf"
sailfishQuasiQuant = "quant.sf/quant.sf"
print("parsing results")
# Gather true results
GroundTruths = []
for i in simIDs:
DF = pd.read_csv('{}/{}.sim.isoforms.results'.format(topdir, i), sep='\t')
DF.drop(set(DF.columns) - set(['transcript_id', 'length', 'count',
'TPM', 'effective_length']), axis=1, inplace=True)
DF.rename(columns={'transcript_id' : 'Name',
'length' : 'Length_truth{}'.format(i),
'count' : 'NumReads_truth{}'.format(i),
'TPM' : 'TPM_truth{}'.format(i),
'effective_length' : 'EffLen_truth{}'.format(i)}, inplace=True)
DF.set_index('Name', inplace=True)
DF.convert_objects(convert_numeric=True)
GroundTruths.append(DF)
SalmonRes = []
# Gather salmon results
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'salmon', salmonQuant])
DF = ParsingUtils.readSalmon(fn, '_{}{}'.format('Salmon', i))
SalmonRes.append(DF)
SailfishRes = []
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'sailfish', sailfishQuant])
DF = ParsingUtils.readSailfish(fn, '_{}{}'.format('Sailfish', i))
SailfishRes.append(DF)
SalmonVBRes = []
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'salmonVB', salmonQuant])
DF = ParsingUtils.readSalmon(fn, '_{}{}'.format('Salmon (VB)', i))
SalmonVBRes.append(DF)
SalmonAlnVBRes = []
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'salmon_alnVB', salmonQuant])
DF = ParsingUtils.readSalmon(fn, '_{}{}'.format('SalmonAln (VB)', i))
SalmonAlnVBRes.append(DF)
SalmonAlnRes = []
# Gather salmon results
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'salmon_aln', salmonQuant])
DF = ParsingUtils.readSalmon(fn, '_{}{}'.format('SalmonAln', i))
SalmonAlnRes.append(DF)
# Gather kallisto results
KallistoRes = []
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'kallisto', kallistoQuant])
DF = ParsingUtils.readKallisto(fn, '_{}{}'.format('Kallisto', i))
KallistoRes.append(DF)
# Gather sailfish quasi results
SailfishQuasiRes = []
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'sailfish_quasi', sailfishQuasiQuant])
DF = ParsingUtils.readSalmon(fn, '_{}{}'.format('Sailfish (Quasi)', i))
SailfishQuasiRes.append(DF)
# Gather eXpress results
expressQuant = "results.xprs"
ExpressRes = []
for i in simIDs:
fn = os.path.sep.join([topdir, str(i), 'express', expressQuant])
DF = ParsingUtils.readExpress(fn, '_{}{}'.format('eXpress', i))
ExpressRes.append(DF)
# Gather StringTie results
#stringtieQuant = "t_data.ctab"
#StringtieRes = []
#for i in simIDs:
# fn = os.path.sep.join([topdir, str(i), 'stringtie', stringtieQuant])
# DF = ParsingUtils.readExpress(fn, '_{}{}'.format('stringtie', i))
# StringtieRes.append(DF)
K = KallistoRes[0].join(KallistoRes[1:])
S = SalmonRes[0].join(SalmonRes[1:])
SVB = SalmonVBRes[0].join(SalmonVBRes[1:])
SA = SalmonAlnRes[0].join(SalmonAlnRes[1:])
SAVB = SalmonAlnVBRes[0].join(SalmonAlnVBRes[1:])
E = ExpressRes[0].join(ExpressRes[1:])
G = GroundTruths[0].join(GroundTruths[1:])
SF = SailfishRes[0].join(SailfishRes[1:])
SFQ = SailfishQuasiRes[0].join(SailfishQuasiRes[1:])
#ST = StringtieRes[0].join(StringtieRes[1:])
# Gather *all* results into a single dataframe
M = G.join(K).join(S).join(SA).join(SVB).join(SAVB).join(E).join(SF).join(SFQ)
methods = ["Salmon", "Salmon (VB)", "SalmonAln", "SalmonAln (VB)", "eXpress", "Kallisto", "Sailfish", "Sailfish (Quasi)"]
# Filter eXpress results
for i in simIDs:
minVal = np.inf
for mn in set(methods) - set(["Truth", "eXpress"]):
print("Method name = {}".format(mn))
newMin = M.loc[M["{}_{}{}".format(measure, mn, i)]>0, "{}_{}{}".format(measure, mn, i)].min()
minVal = min(minVal, newMin)
print("filtering eXpress results < {} {}".format(minVal, measure))
AnalysisUtils.filterValues("{}_{}{}".format(measure, "eXpress", i), M, minVal)
#print("generating relDiff plot")
#rsemRelDiffPlots(M, measure, methods, simIDs, outdir, "MeanAbsRelDiffs{}".format(measure))
#print("generating proportionality plot")
#proportionalityCorrelationPlots(M, measure, methods, simIDs, outdir, "ProportionalityCorrelation{}".format(measure))
print("generating spearman correlation plot")
methods_spearman = ["Sailfish", "eXpress", "Salmon", "SalmonAln"]
spearmanCorrelationPlots(M, measure, methods_spearman, simIDs, outdir, "SpearmanCorrelation{}".format(measure), setxlim=(0.87, 0.93))
methods_spearman_kallisto = ["Kallisto", "Salmon"]
spearmanCorrelationPlots(M, measure, methods_spearman_kallisto, simIDs, outdir, "SpearmanCorrelationKallisto{}".format(measure), setxlim=(0.87, 0.93))
methods_spearman_salmon = ["Salmon (VB)","Salmon", "SalmonAln (VB)", "SalmonAln" ]
spearmanCorrelationPlots(M, measure, methods_spearman_salmon, simIDs, outdir, "SpearmanCorrelationSalmon{}".format(measure), setxlim=(0.91, 0.93), lines=True)
methods_spearman_sailfishquasi = ["Sailfish", "Sailfish (Quasi)", "eXpress", "Salmon", "SalmonAln"]
spearmanCorrelationPlots(M, measure, methods_spearman_sailfishquasi, simIDs, outdir, "SpearmanCorrelationSailfishQuasi{}".format(measure), setxlim=(0.87, 0.93), lines=True)
#print("generating tp relative error plots")
#errorFracPlots(M, measure, methods, simIDs, outdir, "TPErrorFrac{}".format(measure))
#for i in simIDs:
# for m in methods + ['truth']:
# AnalysisUtils.filterValues("NumReads_{}{}".format(m, i), M, 1.0)
print("generating filtered relDiff plot")
#rsemRelDiffPlots(M, measure, methods, simIDs, outdir, "MeanAbsRelDiffs{}Filtered".format(measure))
print("generating filtered proportionality plot")
#proportionalityCorrelationPlots(M, measure, methods, simIDs, outdir, "ProportionalityCorrelation{}Filtered".format(measure))
print("generating spearman correlation plot")
