def negative_diff(b1, b2, thres=-0.3, verbose=True): select = (b1 - b2) < thres if verbose: wzcore.err_print('Selected %d probes' % select.sum()) return b1.loc[select,], b2.loc[select,]
def data_load_samples(samples, probes=None): if not isinstance(samples, list): samples = [samples] data_home = '/Users/wandingzhou/projects/hs-tcga/data/2015_03_05_TCGA_450/dat/' _betases = [] cancer_types = pd.Series() wzcore.err_print_sig() for cancer_type in samples: wzcore.err_print_m(' '+cancer_type) dat_fn = cancer_type+'.pkl' _betas = pd.read_pickle(data_home+'/'+dat_fn) if probes is None: _betases.append(_betas) # = pd.concat([betas, _betas], axis=1) else: _betases.append(_betas.loc[probes,]) # _betas = pd.concat([betas, _betas.loc[probes,]], axis=1) cancer_types = cancer_types.append(pd.Series([cancer_type]*_betas.shape[1], index=_betas.columns)) betas = pd.concat(_betases, axis=1) wzcore.err_print_m('\n') # some cell line sample belong to multiple cancer type, choose the last cancer cancer_types = cancer_types.groupby(level=0).last() betas = betas.groupby(level=0, axis=1).mean() wzcore.err_print('Loaded %d probes and %d samples' % betas.shape) return betas, cancer_types
def mutstat(self, samples, gene, verbose=True): t = [] cnt = 0 cnt1 = 0 if gene not in self.genes: wzcore.err_print('Gene ID not identified %s' % gene) return None for s in samples: s = s[:12] if s in self.sample2muts: mut = False for g, mt, mt1 in self.sample2muts[s]: if g == gene: mut = True cnt1 += 1 break t.append(mut) cnt += 1 else: t.append(np.nan) if verbose: wzcore.err_print('Identified info from %d/%d samples (%d muts).' % (cnt, len(samples), cnt1)) return pd.Series(t, index=samples)
def data_load_tissue(source): if source == 'ESC': # this has H1, H9, H9ESC, HUES6 and ICM betas = pd.read_pickle('/Users/wandingzhou/projects/hs-tcga/2016_03_29_TGCT/data/ESC_and_ICM.pkl') if source == 'PGC': # this has PGC and AGC betas = pd.read_pickle('/Users/wandingzhou/projects/hs-tcga/2016_03_29_TGCT/data/PGC.pkl') if source == 'Laird': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_06_03_AML_normal_sorted/GSE49618_betas.tsv') samples = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_06_03_AML_normal_sorted/samples', index_col='barcode') betas.columns = betas.columns.map(lambda x: 'blood_'+samples.loc[x.split('_')[0],'name']) if source == 'Encode': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE40699_ENCODE/GSE40699_betas.tsv') samples = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE40699_ENCODE/samples', index_col='barcode') betas.columns = samples.loc[betas.columns.map(lambda x:x.split('_',2)[2]), 'short']+'_'+samples.loc[betas.columns.map(lambda x:x.split('_',2)[2]), 'cellline'] if source == 'Bonder': # muscle and fat betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/Bonder2014_BMCGenomics/GSE61454_severely_obsese/betas.tsv') samples = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/Bonder2014_BMCGenomics/GSE61454_severely_obsese/samples',header=None,index_col=0,names=['barcode','sample']) betas.columns = samples.loc[betas.columns.map(lambda x:x.split('_',1)[0]),'sample']+"_"+betas.columns.map(lambda x:x.split('_',1)[0]) betas = betas.loc[:,~betas.columns.map(lambda x:x.startswith('Liver'))] # exclude liver if source == 'Slieker': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE48472_Slieker_2013_EpigeneticsAndChromatin/betas.tsv') samples = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE48472_Slieker_2013_EpigeneticsAndChromatin/samples',header=None,index_col=0,names=['barcode','sample']) betas.columns = samples.loc[betas.columns.map(lambda x:x.split('_',1)[0]),'sample']+"_"+betas.columns.map(lambda x:x.split('_',1)[0]) if source == 'Wong': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_06_03_AlexWong_skin/AlexWong_skin_betas.tsv') samples = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_06_03_AlexWong_skin/samples', index_col='barcode') betas.columns = 'skin_'+betas.columns.to_series() if source == 'Lawlor1133': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_06_03_Lawlor_tumor_lungfibroblast/s1133_betas.tsv') names = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_06_03_Lawlor_tumor_lungfibroblast/1133samples.csv.unix', index_col='Complete_Barcode', sep='\t') betas.columns = names.loc[betas.columns,'GROUP_NAME']+'_'+betas.columns if source == 'Guintivano': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE41826_Brain/betas.tsv') mask_snp_probes(betas) names = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE41826_Brain/samples.csv',index_col='barcode') betas.columns = 'brain_'+names.loc[betas.columns,'sample'].map(str)+'_'+betas.columns if source == 'Wagner': betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_03_18_tumor_purity/GSE52025_Wagner_fibroblast/GSE52025_betas.tsv',sep='\t') betas.columns = 'fibroblast_'+betas.columns.map(str) if source == 'Reinus': # blood betas = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_04_10_sorted_cell_population/blood_beta.tsv', sep='\t') names = pd.read_table('/Users/wandingzhou/projects/hs-tcga/data/2015_04_10_sorted_cell_population/Sorted_Blood/sample_sheet_IDAT.csv.unix.tsv',index_col='barcode') betas.columns = names.loc[betas.columns,'Type']+'-'+betas.columns wzcore.err_print("Loaded %d samples" % betas.shape[1]) return betas
def data_load_rnaseq_samples(_cancer_types, genes=None): if not isinstance(_cancer_types, list): _cancer_types = [_cancer_types] data_home = '/Users/wandingzhou/projects/hs-tcga/data/2015_04_30_TCGA_rnaseq/dat/' _rsems = [] cancer_types = pd.Series() wzcore.err_print_sig() for cancer_type in _cancer_types: wzcore.err_print_m(' '+cancer_type) dat_fn = cancer_type+'.pkl' _rsem = pd.read_pickle(data_home+'/'+dat_fn) if genes is None: _rsems.append(_rsem) else: _rsems.append(_rsem.loc[genes,]) cancer_types = cancer_types.append(pd.Series([cancer_type]*_rsem.shape[1], index=_rsem.columns)) rsems = pd.concat(_rsems, axis=1) wzcore.err_print_m('\n') cancer_types = cancer_types.groupby(level=0).last() wzcore.err_print('Loaded %d genes and %d samples' % rsems.shape) return rsems, cancer_types
def split_tumor_normal(df): dft = df.loc[:,df.columns.map(lambda x:x[13]=='0')] dfn = df.loc[:,df.columns.map(lambda x:x[13]=='1')] dfc = df.loc[:,df.columns.map(lambda x:x[13]=='2')] wzcore.err_print('Found %d tumor, %d normal, %d cell line and %d others.' % (dft.shape[1], dfn.shape[1], dfc.shape[1], df.shape[1]-dft.shape[1]-dfn.shape[1]-dfc.shape[1])) return dft, dfn, dfc
def nonuniform(df, maxbeta=0.7, minbeta=0.3, maxsupp=0.95): maxsuppn = float(df.shape[1]) * maxsupp def _is_nonuniform(row): return not (((row > maxbeta).sum() > maxsuppn) or ((row < minbeta).sum() > maxsuppn)) dfv = df[df.apply(_is_nonuniform, axis=1)] wzcore.err_print('Selected %d nonuniform probes from %d' % (dfv.shape[0], df.shape[0])) return dfv
def data_create_tissue_sample(): samples = pd.read_table('/Users/wandingzhou/projects/hs-tcga/2015_07_14_purity/2015_07_14_sample_select.txt', index_col="sample") betas = pd.DataFrame() for k, v in samples.sort('source').groupby('source'): _betas = data_load_tissue(v['source'][0]) _betas2 = _betas[_betas.columns.intersection(v.index)] betas = pd.concat([betas, _betas2], axis=1) wzcore.err_print("From %s chose %d samples" % (v['source'][0], _betas2.shape[1])) wzcore.err_print("Loaded %d samples" % betas.shape[1]) return betas, samples
def take_segment_mean(df, probe2seg, min_support=10, quiet=False): _df_seg = df.copy() if isinstance(_df_seg, pd.Series): _df_seg = _df_seg.to_frame() _df_seg['seg'] = probe2seg.loc[df.index] _df_seg_gb = _df_seg.groupby('seg') df_seg_count = _df_seg_gb.count().iloc[:,0] df_seg_mean = _df_seg_gb.mean()[df_seg_count >= min_support] if not quiet: wzcore.err_print('There are %d segments well supported.' % df_seg_mean.shape[0]) return df_seg_mean
def associate_continuous_outliers(self, outlier_fn, fw=20.0, printn=10): balanced_up = [((pval+foldc*fw)/2.0,g,pval,foldc) for pval, foldc, g in zip(self.pvals, self.foldcs, self.genes_select) if (not pd.isnull(pval)) and (not pd.isnull(foldc))] balanced_dw = [((pval-foldc*fw)/2.0,g,pval,foldc) for pval, foldc, g in zip(self.pvals, self.foldcs, self.genes_select) if (not pd.isnull(pval)) and (not pd.isnull(foldc))] import wzplotlib tss = self.ts.copy() tss.sort() cbs = [wzplotlib.WZCbar(tss, continuous=True)] balanced_up.sort(reverse=True) wzcore.err_print('upside:') for i in xrange(min(printn, len(self.genes_select))): b, g, pval, foldc = balanced_up[i] cbs.append(wzplotlib.WZCbar(self.mutstat(tss.index, g, verbose=False)[tss.index], title=g+' [up]')) wzcore.err_print('%s\tpval:%1.2f\tfoldc:%1.2f\tbalanced:%1.2f' % (g, pval, foldc, b)) balanced_dw.sort(reverse=True) wzcore.err_print('\ndownside:') for i in xrange(min(printn, len(self.genes_select))): b, g, pval, foldc = balanced_dw[i] cbs.append(wzplotlib.WZCbar(self.mutstat(tss.index, g, verbose=False)[tss.index], title=g+' [down]')) wzcore.err_print('%s\tpval:%1.2f\tfoldc:%1.2f\tbalanced:%1.2f' % (g, pval, foldc, b)) wzplotlib.row_stack_layout(cbs, figfile=outlier_fn)
def filter_by_purity(betas, min_purity=0.8, keep_normal=True): Hui_annot = pd.read_csv('/Users/wandingzhou/projects/hs-tcga/data/2015_03_23_Hui_annotation/sampleAnnotSubWB20130619.tsv', index_col=1, error_bad_lines=False, sep='\t') # print Hui_annot.columns[Hui_annot.columns.map(lambda x: np.bool(re.search('purity', x)))] # there are two purities, ABSOLUTE.purity and abs.purity, ABSOLUTE.purity has more value (4776 vs 4468) # the two values agree on overlap # I hereby use ABSOLUTE.purity # 4239 samples with purity estimates purity_anno = Hui_annot['ABSOLUTE.purity'] purity_anno = purity_anno[(purity_anno.notnull()) & (purity_anno != 1.0)] sample_is_pure = betas.columns.map(lambda x: (x in purity_anno and purity_anno[x] > min_purity) or x[13]=='1') betasv = betas.loc[:,sample_is_pure].copy() wzcore.err_print('Selected %d pure samples (>%1.2f) from %d samples.' % (betasv.shape[1], min_purity, betas.shape[1])) return betasv
def load_te_and_seqs( rmskbed='/Users/wandingzhou/projects/pj-mm/2015-04-23-alu/rmsk.bed.gz', load_seq=False, tetype=None, tetype2=None, tetype3=None): refgenome = faidx.RefGenome('/Users/wandingzhou/references/hg19/hg19.fa') tes = {} wzcore.err_print_sig() for i, line in enumerate(wzcore.opengz(rmskbed)): if i % 100000 == 0: wzcore.err_print_m(' %d' % i) fields = line.strip().split('\t') te = TE() te.chrm = fields[0] if te.chrm.find('_') > 0: continue te.beg = int(fields[1]) te.end = int(fields[2]) te.rmskbed = rmskbed te.strand = fields[3] te.tetype = fields[4] te.tetype2 = fields[5] te.tetype3 = fields[6] if tetype is not None and te.tetype != tetype: continue if tetype2 is not None and te.tetype2 != tetype2: continue if tetype3 is not None and te.tetype3 != tetype3: continue if load_seq: try: _te_load_seqs(refgenome, te) except IndexError: # TE at chromosome boundaries, ignore # te.seq == None pass tes[(te.chrm, te.beg, te.end)] = te wzcore.err_print_m('\n') wzcore.err_print('Loaded %d TEs' % len(tes)) return tes
def associate_continuous_outliers(self, fn, fw=10, printn=10): import matplotlib.pyplot as plt plt.figure(figsize=(10,10)) plt.subplots_adjust(hspace=1) toplist = sorted(zip(self.pvals, self.rhos, self.genes_select), reverse=True) n = min(printn, len(toplist)) for i in xrange(n): pval, rho, gene = toplist[i] wzcore.err_print('%s\tpval:%1.2f\trho:%1.2f' % (gene, pval, rho)) plt.subplot(5,2,i+1) _dgene, _ts = self.expstat(self.ts, gene) plt.scatter(np.log2(_dgene), _ts, edgecolor='none', alpha=0.5, s=4) plt.xlabel('log2('+gene+' %1.2f)' % rho) plt.savefig(fn, bbox_inches='tight')
def probe_select_pairwise(betas, v1, v2, mindelta=0.5, upq=0.75, loq=0.25): inind1 = betas.columns.isin(v1.index) inind2 = betas.columns.isin(v2.index) select1 = [] select2 = [] for i, row in betas.iterrows(): c1 = row[inind1] c2 = row[inind2] if c2.quantile(loq) - c1.quantile(upq) > mindelta: # hypo select1.append(i) if c1.quantile(loq) - c2.quantile(upq) > mindelta: # hyper select2.append(i) wzcore.err_print("selected %d hypo and %d hyper." % (len(select1), len(select2))) return select1, select2
def __init__(self): self.sample2muts = {} self.genes = set() with open('/Users/wandingzhou/projects/hs-tcga/data/2015_06_17_TCGA_mutations/merged_maf') as fh: for line in fh: fields = line.strip().split() if fields[2] in ['Silent', 'RNA']: continue sample = fields[4][:12] if sample not in self.sample2muts: self.sample2muts[sample] = [] mut = fields[1] self.sample2muts[sample].append((mut, fields[2], fields[3])) self.genes.add(mut) wzcore.err_print('Loaded %d genes and %d samples' % (len(self.genes), len(self.sample2muts))) return
def load_cgi_and_seqs(): cgis = [] refgenome = faidx.RefGenome('/Users/wandingzhou/references/hg19/hg19.fa') for line in wzcore.opengz( '/Users/wandingzhou/projects/hs-tcga/data/2015_03_24_cpg_island/TakaiJones/takai.jones.strict.bed.gz' ): fields = line.strip().split('\t') cgi = CGI() cgi.chrm = fields[0] cgi.beg = int(fields[1]) cgi.end = int(fields[2]) cgi.cgitype = fields[3] cgi.seq = refgenome.fetch_sequence(cgi.chrm, cgi.seq_beg(), cgi.seq_end()).upper() cgis.append(cgi) wzcore.err_print('Loaded %d CGIs' % len(cgis)) return cgis
def probe_select1(betas, v, upq=0.75, loq=0.25, mindelta=0.1): inind = betas.columns.isin(v.index) _select1 = [] _select2 = [] hyperrows = [] hyporows = [] for i, row in betas.iterrows(): c = row[inind] cb = row[~inind] hyperrows.append((c.min()-cb.quantile(upq),i)) hyporows.append((cb.quantile(loq)-c.max(),i)) _select1 = [(i,j) for i,j in sorted(hyporows, reverse=True)[:100] if i>mindelta] _select2 = [(i,j) for i,j in sorted(hyperrows, reverse=True)[:100] if i>mindelta] wzcore.err_print("selected %d hypo and %d hyper probes." % (len(_select1), len(_select2))) return list(set([j for i,j in _select1]) | set([j for i,j in _select2]))
def polarized(df, upthres=0.7,dwthres=0.3, kind='any', minsupp=1): """ select probes that are methylated in some samples and unmethylated in others "any" means more than minsupp probes have hypo and hyper meth "all" all probes have either hypo or hyper meth """ def _isvar(row): hi = (row > upthres).sum() lo = (row < dwthres).sum() if hi >= minsupp and lo >= minsupp: if kind == 'any': return True elif kind == 'all' and hi + lo == row.shape[0]: return True else: return False else: return False dfv = df[df.apply(_isvar, axis=1)] wzcore.err_print('Selected %d variable probes from %d samples' % dfv.shape) return dfv
def get_cph(df): df = df.loc[df.index.str(startswith('ch'))] wzcore.err_print('Get %d CpH probes.' % df.shape[0]) return df
def clean_450k(df, nahow='strong', probe_fn='/Users/wandingzhou/projects/hs-tcga/data/2015_03_05_TCGA_450/450k_probes', verbose=True): probe_loc = pd.read_table(probe_fn,index_col=3,header=None, names=['chrm','beg','end','gene']) if verbose: if len(df.shape) == 1: wzcore.err_print("Before: %d probes" % df.shape[0]) else: wzcore.err_print("Before: %d probes and %d samples" % df.shape) # remove X,Y chromosome wzcore.err_print('Removing sex chromosomes') df = df[(~probe_loc.chrm.isin(['chrX','chrY']))[df.index]] wzcore.err_print('Removing cph and rs probes') df = df.loc[df.index.str.startswith('cg')] # remove NA wzcore.err_print('Removing NA probes') if nahow == 'strong': df = df.dropna(how='any') if nahow == 'weak': df = df.dropna(how='all') if verbose: if len(df.shape) == 1: wzcore.err_print("After: %d probes after removal" % df.shape[0]) else: wzcore.err_print("After: %d probes and %d samples" % df.shape) return df