def estimateS(y): eps=1e-3 y[0]=y[0].apply(lambda x: min(1-eps,max(x,eps))) y[59]=y[59].apply(lambda x: min(1-eps,max(x,eps))) s=(2./59 * (utl.logit(y[59])-utl.logit(y[0]))).rename('s') return s import popgen.Run.TimeSeries.RealData.Utils as rutl import scipy as sc import popgen.Plots as pplt import popgen.Run.TimeSeries.RealData.Utils as rutl import popgen.Run.TimeSeries.RealData.Data as dta import popgen.TimeSeries.Markov as mkv S=np.arange(-1,1,0.05).round(2);chroms=['2L','2R','3L','3R','X'] pd.read_pickle('/home/arya/out/real/HMM1x/h5.000000E-01.df').loc[i] scores = utl.getEuChromatin(rutl.loadScores(skipHetChroms=True)).loc[chroms].rename('score') cdAll=utl.getEuChromatin(pd.read_pickle('/home/arya/out/real/CD.F59.df').loc[chroms]) freq=lambda x:x.xs('C',level='READ',axis=1).sum(1)/x.xs('D',level='READ',axis=1).sum(1) s=estimateS(cdAll.groupby(axis=1,level='GEN').apply(freq)[[0,37,59]]) x=pd.read_pickle('/home/arya/out/real/HMM1x/h5.000000E-01.df').loc[chroms,0.5] pplt.Manhattan(utl.zpvalgenome(utl.scanGenome(utl.zpvalgenome2tail(s)))) (x.s*(x.alt-x.null)).hist(bins=100) D=cdAll.xs('D',axis=1,level='READ') d=D.median(1).rename('d') f=lambda x:(x.alt-x.null) pplt.Manhattan(utl.scanGenome(x2p(f(x)))) x2p=lambda X2: -pd.Series(1 - sc.stats.chi2.cdf(X2, 1),index=X2.index).apply(np.log) y=(f(pd.read_pickle('/home/arya/out/real/HMM/h5.000000E-01.df')[0.5]).loc[chroms].rename('y')*pd.read_pickle('/home/arya/out/real/HMM/h5.000000E-01.df')[0.5].s).dropna() y.sort_values() y=utl.zpvalgenome(pd.read_pickle('/home/arya/out/real/HMM/h5.000000E-01.df')[0.5].s.loc[chroms])
def Final(): ############ preparing data def saveGOTex(df): name = np.unique(df.index)[0] print '*' * 80, name df = df.sort_values('-log($p$-value)', ascending=False) df['Rank'] = range(1, df.shape[0] + 1); df = df.iloc[:, [6] + range(6)] path = utl.paperPath + '/tables/{}.tex'.format(name); df.to_csv(path.replace('.tex', '.csv').replace('/tables/', '/data/')) utl.DataframetolaTexTable(df.iloc[:, :-1], alignment=['c', 'c', 'p{3in}', 'c', 'c', 'c'], fname=path) goPvalue = lambda x: utl.getPvalFisher(AllGenes=allVariantGenes.values, putativeList=x.values, myList=g.index.values) unpackp = lambda x: [min(6, np.round(x[0], 1)), x[1].loc['Putative', 'myList']] # Score = lambda x,f:f(scores.loc[x.CHROM][(scores.loc[x.CHROM].index>=x.start)&(scores.loc[x.CHROM].index<=x.end)]) sort = lambda df: pd.concat( [df[df.index.get_level_values('CHROM') == ch] for ch in ['X', '2L', '2R', '3L', '3R']]).rename( columns={'H': r'$\mathcal{H}^+$', 'M': 'Num. of Variants'}) Genes = loadGeneData().reset_index().set_index('GO') Genes = Genes.loc[ (Genes['FBgn'].groupby(level=0).apply(lambda x: len(x.unique())) > 2).replace({False: None}).dropna().index] scores = utl.getEuChromatin(rutl.loadScores(skipHetChroms=True)) ann = pd.DataFrame(scores).join(loadANN(), how='inner') allVariantGenes = ann['Gene_ID'].drop_duplicates() # f=lambda x: x[x>=x.quantile(0.9)].mean() # geneScores=ann.reset_index().set_index('Gene_ID')[['CHROM','POS',0]].drop_duplicates().groupby(level=0)[0].apply(f) ############ computing candidate regions scan = utl.scanGenome(scores.abs(), {'H': lambda x: x.abs().mean(), 'M': lambda x: x.size}, winSize=30000) o = utl.localOutliers(scan.H, q=0.99); o = scan.loc[o.index] fig = plt.figure(figsize=(7, 2.5), dpi=300); pplt.Manhattan(data=sort(scan), Outliers=sort(o), fig=fig, markerSize=2, ticksize=8, sortedAlready=True); [pplt.setSize(ax, 6) for ax in fig.get_axes()]; pplt.annotate('(A)', ax=fig.axes[0], fontsize=8) pplt.annotate('(B)', ax=fig.axes[1], fontsize=8) plt.gcf().subplots_adjust(bottom=0.15); pplt.savefig('manhattan', 300) plt.savefig(utl.paperFiguresPath + 'manhattan.pdf') regions = utl.BED.getIntervals(o.H, padding=30000); print regions.shape intervalGenes = utl.BED.intersection(ann, regions).name.drop_duplicates().reset_index().set_index('name'); print intervalGenes.size g = intervalGenes; # intervalGenes # g=g[g>=g.quantile(0.)]; print g.size df = Genes.groupby(level=0).apply(lambda x: pd.DataFrame( [x.name, x.term.iloc[0]] + unpackp(goPvalue(x.FBgn.drop_duplicates())) + [x.ontology.iloc[0], x.FBgn.unique().size] + [ np.intersect1d(x.values, g.index.values)], index=['GO ID', 'GO Term', '-log($p$-value)', 'Hits', 'Ontology', 'Num of Genes', 'Genes']).T) df = df[(df['-log($p$-value)'] >= 3) & (df.Hits >= 3)] df['-log($p$-value)'] = df['-log($p$-value)'].astype(str) df = df.set_index('Ontology') df.groupby(level=0).apply(saveGOTex); print df tempGenes = Genes.reset_index().set_index('FBgn').loc[ np.append(df.set_index('GO ID').loc['GO:0009631'].Genes, df.set_index('GO ID').loc['GO:0009408'].Genes)][ ['term', 'name', 'GO']].reset_index().set_index('GO').loc[['GO:0009631', 'GO:0009408']].drop_duplicates() tempGenes.columns = ['FlyBase ID', 'GO Term', 'Gene Name'] utl.DataframetolaTexTable(tempGenes, fname=utl.paperPath + '/tables/{}.tex'.format('tempGenes'), alignment=['l', 'l', 'l']) regions.to_csv(utl.paperPath + 'data/intervals.csv') snps = utl.BED.intersection(scores.reset_index(), regions, 0); snps['POS'] = snps.start; snps.set_index('POS', append=True, inplace=True) snps = snps['name'].astype(float).reset_index().drop_duplicates().set_index(['CHROM', 'POS']).name def ff(x): y = utl.BED.intersection(scores.reset_index(), x, 0).rename(columns={'start': 'POS'}).set_index('POS', append=True).name.astype( float) y = y[y > 0] y = y[y >= y.quantile(0.9)] print x['len'].iloc[0], y.size return y cands = regions.reset_index().groupby(level=0).apply(ff).reset_index(level=0).name cands.sort_index().reset_index().drop_duplicates().dropna().to_csv(utl.outpath + 'real/gowinda/cands.final.txt', sep='\t', header=None, index=False) scores.sort_index().reset_index().drop_duplicates().dropna().to_csv(utl.outpath + 'real/gowinda/allsnps.txt', sep='\t', header=None, index=False) name = 'cands.final.out.tsv' gowinda = pd.read_csv('/home/arya/out/real/gowinda/{}'.format(name), sep='\t', header=None)[[0, 4, 5, 6, 7, 8, 9]] gowinda.columns = ['GO ID', '-log($p$-value)', 'Hits', 'Num of Genes', 'Total Genes', 'GO Term', 'Genes'] gowinda = gowinda[gowinda.Hits >= 3] gowinda['-log($p$-value)'] = -gowinda['-log($p$-value)'].apply(np.log10).round(1) gowinda.to_csv(utl.paperPath + 'data/gowinda.all.tsv', sep='\t') bp = gowinda.set_index('GO ID').loc[ Genes[Genes.ontology == 'biological_process'].index.unique().rename('GO ID')].dropna() bp.to_csv(utl.paperPath + 'data/gowinda.bp.tsv', sep='\t') utl.DataframetolaTexTable(bp.reset_index()[['GO ID', 'GO Term', '-log($p$-value)']], alignment=['c', 'p{4in}', 'c'], fname=utl.paperPath + 'tables/gowinda.tex') map(len, (Genes.index.unique(), bp.index.unique(), df.loc['biological_process']['GO ID'].unique())), len( np.intersect1d(bp.index.unique(), df['GO ID'].unique())) pval = utl.getPvalFisher(Genes[Genes.ontology == 'biological_process'].index.unique(), bp.index.unique(), df.loc['biological_process']['GO ID'].unique()) print pval stats = pd.Series(None, name='Value') stats['Num. of Vatiants'] = scores.size stats['Num. of Candidate Intervals'] = regions.shape[0] stats['Total Num. of Genes'] = loadGeneCoordinates().shape[0] stats['Num. of Variant Genes'] = ann['Gene_ID'].unique().shape[0] stats['Num. of Genes within Candidate Intervals'] = intervalGenes.shape[0] stats['Total Num. of GO'] = len(loadGeneData().index.unique()) stats['Num. of GO with 3 or More Genes'] = len(Genes.index.unique()) stats['Num. of Candidate Variants for Gowinda'] = cands.size stats = stats.apply(lambda x: '{:,.0f}'.format(x)) stats.index.name = 'Statistic' print stats utl.DataframetolaTexTable(stats.reset_index(), fname=utl.paperPath + 'tables/stats.tex', alignment=['l', 'r'])