def compute_cross_de(rnaseq_obj, main_pids, additional_pids, external_references=(('GIBCO', 'NSC'), ), lfc=1, fdr=0.01, method='QLGLM'): """ Compute DE between every patient GBM sample and every _other_ healthy patient sample, in addition to paired DE. We can also include one or more external references (e.g. Gibco, the default). :param rnaseq_obj: :param main_pids: The PIDs of the patients in the study :param additional_pids: The PIDs of the additional iNSC lines to include. These will be treated as additional reference lines. :param external_references: :param lfc: :param fdr: :param method: :param njob: :return: """ if method not in {'QLGLM', 'GLM', 'exact'}: raise NotImplementedError("Unsupported method.") de = {} for pid in main_pids: # cross comparison for pid2 in list(main_pids) + list(additional_pids): the_idx = (rnaseq_obj.meta.index.str.contains(pid) & (rnaseq_obj.meta.loc[:, 'type'] == 'GBM')) | \ (rnaseq_obj.meta.index.str.contains(pid2) & (rnaseq_obj.meta.loc[:, 'type'] == 'iNSC')) the_data = rnaseq_obj.data.loc[:, the_idx] the_data = filter.filter_by_cpm(the_data, min_n_samples=1) the_groups = rnaseq_obj.meta.loc[the_idx, 'type'].values the_comparison = ['GBM', 'iNSC'] de[(pid, pid2)] = run_one_de(the_data, the_groups, the_comparison, lfc=lfc, fdr=fdr, method=method) # external reference comparison for er, er_type in external_references: the_idx = (rnaseq_obj.meta.index.str.contains(pid) & (rnaseq_obj.meta.loc[:, 'type'] == 'GBM')) | \ (rnaseq_obj.meta.index.str.contains(er) & (rnaseq_obj.meta.loc[:, 'type'] == er_type)) the_data = rnaseq_obj.data.loc[:, the_idx] the_data = filter.filter_by_cpm(the_data, min_n_samples=1) the_groups = rnaseq_obj.meta.loc[the_idx, 'type'].values the_comparison = ['GBM', er_type] de[(pid, er)] = run_one_de(the_data, the_groups, the_comparison, lfc=lfc, fdr=fdr, method=method) return de
rna_ff_dat = rna_ff_obj.data.copy() rna_cc_dat = rna_cc_obj.data.copy() # update labels rna_ff_dat.columns = ["GBM%s" % t for t in p_id] cols = [] for t in rna_cc_dat.columns: p, q = t.replace('GBM', 'GIC').split('_') q = q.replace('n', ' & P') cols.append("%s (%s)" % (p, q)) rna_cc_dat.columns = cols # filter low expression out rna_ff_dat = filter.filter_by_cpm(rna_ff_dat, min_cpm=min_tpm, min_n_samples=2) rna_cc_dat = filter.filter_by_cpm(rna_cc_dat, min_cpm=min_tpm, min_n_samples=2) # reduce to matching probes probes = rna_cc_dat.index.intersection(rna_ff_dat.index) if remove_mt: probes = probes[~probes.isin(mt_ens)] rna_ff_dat = np.log2(rna_ff_dat.loc[probes] + eps) rna_cc_dat = np.log2(rna_cc_dat.loc[probes] + eps) # QN
polya_nsc_meta.index] assigned_sum = polya_nsc_data.sum() unassigned_sum = polya_nsc_unassigned.drop('N_unmapped').sum() polya_pct_assigned = assigned_sum / (assigned_sum + unassigned_sum) * 100. print "Poly(A) samples % assigned" print polya_pct_assigned # combine data then eliminate genes that are not expressed ss2_data = ss2_obj.data ss2_data.columns = ["%s_smartseq" % t for t in ss2_data.columns] polya_nsc_data = polya_obj.data data = pd.concat((ss2_data, polya_nsc_data), axis=1) data = filter.filter_by_cpm(data, min_cpm=min_cpm, min_n_samples=1) # TMM normed version data_n = transformations.edger_tmm_normalisation_cpm(data) # plot the CDF of the log2(CPM) values # for this purpose, we need to filter the CPM values for each col separately x_cdf = np.linspace(-5, 15, 500) log_cpm_ecdf = {} fig = plt.figure() ax = fig.add_subplot(111) ss_nsc_lbl = False ss_opc_lbl = False pa_lbl = False
if __name__ == '__main__': lfc = 1 fdr = 0.01 outdir = output.unique_output_dir("paired_rnaseq") # RTK II samples # pids = ['017', '050', '054', '061'] # all n=2 samples pids = ['018', '044', '049', '050', '052', '054', '061'] # all samples # pids = [t for t in rnaseq_data.PATIENT_LOOKUP_STAR if t != 'GIBCO'] obj = rnaseq_data.load_by_patient(pids, annotate_by='Ensembl Gene ID') # discard unmapped, etc obj.data = obj.data.loc[obj.data.index.str.contains('ENSG')] dat_filt = filter_by_cpm(obj.data, min_n_samples=2) de = {} de_up = {} de_down = {} de_counts = {} de_counts_up = {} de_counts_down = {} de_matched = {} de_gibco = {} # all gene lists combined in one file (one sheet per list) xl_writer = pd.ExcelWriter( os.path.join(outdir, "individual_gene_lists.xlsx")) for pid in pids:
return cor, pval if __name__ == "__main__": min_tpm = 1. eps = 0.01 outdir = output.unique_output_dir() # load just 1st lane obj = loader.load_references('wtchg_p180443/180911_K00150_0372_AHWV7TBBXX', tax_id=10090, source='salmon') ix = (obj.meta.species == 'mouse') | (obj.meta.index.str.contains(r'[iI]MGL')) dat = obj.data.loc[:, ix] dat_filt = filter.filter_by_cpm(dat, min_cpm=min_tpm, unless_cpm_gt=10.) log_dat_filt = np.log2(dat_filt + eps) # correlation clustermap row_colours = pd.DataFrame('g', index=log_dat_filt.columns, columns=['Sample type']) row_colours.loc[row_colours.index.str.contains('mDURA')] = 'k' row_colours.loc[row_colours.index.str.contains( 'mDURA5_NSCmus_N3BE50.2')] = 'y' row_colours.loc[row_colours.index.str.contains('mDURA6_NSCmus')] = 'y' # Spearman distance cor, pval = pairwise_correlation(log_dat_filt.transpose(), method='spearman')
dat_pt = pd.read_csv(dat_fn, header=0, index_col=0) dat_hn = pd.read_csv(dat_hn_fn, header=0, index_col=0) dat_pt.columns = [t[:12] for t in dat_pt.columns] dat_pt = dat_pt.loc[:, ~dat_pt.columns.duplicated()] meta = meta.loc[dat_pt.columns] meta = meta.loc[~meta.index.duplicated()] dat_hn.columns = ["%s_normal" % t[:12] for t in dat_hn.columns] # filter samples keep = (meta['gcimp_methylation'] == 'non-G-CIMP') & (meta['idh1_status'] == 'WT') dat_pt = dat_pt.loc[:, keep] all_dat = pd.concat((dat_pt, dat_hn), axis=1) all_dat = filter.filter_by_cpm(all_dat, min_n_samples=3, min_cpm=1) groups = ['GBM'] * dat_pt.shape[1] + ['control'] * dat_hn.shape[1] de_res = differential_expression.run_one_de(all_dat, groups, ('GBM', 'control'), **de_params) # de_res.to_excel(os.path.join(outdir, 'tcga_primary_tumour_vs_normal_solid.xlsx')) de_res_full = differential_expression.run_one_de(all_dat, groups, ('GBM', 'control'), return_full=True, **de_params) de_res_full.to_excel(
0, 'patient_id', obj.meta.index.str.replace(r'(GBM|DURA)(?P<pid>[0-9]{3}).*', '\g<pid>')) cmap = common.get_best_cmap(len(consts.PIDS)) scatter_colours = dict(zip(consts.PIDS, cmap)) scatter_markers = {'GBM': 's', 'iNSC': 'o'} # scaling parameter applied during SVD scale_preserved = 0.05 sample_colours = obj.meta.patient_id.map(scatter_colours.get).to_dict() sample_markers = obj.meta.type.map(scatter_markers.get).to_dict() dat = filter.filter_by_cpm(obj.data, min_n_samples=2) # TODO: include VST or similar here dat = np.log(dat + eps) # copy of dat with gene symbols dat_with_gs = dat.copy() general.add_gene_symbols_to_ensembl_data(dat_with_gs) # fill back in with ENS where no gene symbol is available dat_with_gs.loc[dat_with_gs['Gene Symbol'].isnull(), 'Gene Symbol'] = dat_with_gs.index[ dat_with_gs['Gene Symbol'].isnull()] # recreate Sven's original (unweighted) plot # we're not going to use this, it's just for validation / reproducibility fig, ax, _ = plot_biplot(dat, obj.meta, (0, 1), scatter_colours,
from utils import output import os import numpy as np if __name__ == "__main__": min_cpm = 1 obj = loader.load_by_patient('all', type='ffpe') samples = [ 'NH15_1661DEF2C', 'NH15_1877_SP1C', 'NH15_2101_DEF1A', 'NH16_270_DEF1Ereplacement', 'NH16_616DEF1B', 'NH16_677_SP1A', 'NH16_1574DEF1A', 'NH16_1976_DEF1Areplacement', 'NH16_2063_DEF1Areplacement', 'NH16_2214DEF1A', 'NH16_2255DEF1B2', 'NH16_2806DEF3A1', ] # remove duplicates dat = obj.data.loc[:, samples] dat = filter.filter_by_cpm(dat, min_cpm=min_cpm, min_n_samples=1) cpm = (dat + 1).divide(dat.sum() + 1, axis=1) * 1e6 general.add_gene_symbols_to_ensembl_data(cpm) outdir = output.unique_output_dir("ffpe_logcpm_values") cpm.to_excel(os.path.join(outdir, 'cpm_ffpe_filtered.xlsx'))
source='salmon', tax_id=10090) # dump to file for sharing dat = obj_salmon.data.copy() general.add_gene_symbols_to_ensembl_data(dat, tax_id=10090) dat.to_excel(os.path.join(outdir, 'salmon_tpm_all_data.xlsx')) dat = obj_star.data.copy() general.add_gene_symbols_to_ensembl_data(dat, tax_id=10090) dat.to_excel(os.path.join(outdir, 'star_counts_all_data.xlsx')) # load Bowman data # plots with only our samples dat = filter.filter_by_cpm(obj_salmon.data, min_cpm=min_cpm, min_n_samples=2) log_dat = np.log10(obj_salmon.data + eps) # ECDF ax = rnaseq.log_cpm_ecdf_plot(dat, units='tpm', min_cpm=min_cpm) ax.figure.set_size_inches(6, 4) ax.figure.tight_layout() ax.figure.savefig(os.path.join(outdir, "cdf_our_samples.png"), dpi=200) # PCA colour_subgroups = obj_salmon.meta.treatment cmap = collections.OrderedDict( zip( colour_subgroups,
data = pd.concat((loader_hgic.data.loc[genes], h9_data), axis=1) meta = pd.concat((loader_hgic.meta, pd.DataFrame(h9_meta).transpose()), axis=0) matches = ('018', '019', '024', '026', '030', '031', '044', 'GIBCO', 'H9') col_colours = clustering.generate_colour_map_dict(meta, 'sample', matches, label='Patient', non_matching='gray') # filter data = filter.filter_by_cpm(data, min_cpm=1, min_n_samples=3, unless_cpm_gt=10) # data = filter.filter_by_cpm(data, min_cpm=1, min_n_samples=3, unless_cpm_gt=None) # normalise by read count cpm = (data + 1).divide((data + 1).sum(axis=0), axis=1) * 1e6 # transform log_data = np.log2(cpm) vst_data = variance_stabilizing_transform(cpm) mad_log_srt = median_absolute_deviation(log_data).sort_values( ascending=False) mad_vst_srt = median_absolute_deviation(vst_data).sort_values( ascending=False) for NGENE in [500, 1000, 1500, 2000, 2500]:
import numpy as np from utils import output import os import pandas as pd from matplotlib import pyplot as plt if __name__ == "__main__": eps = 1e-2 outdir = output.unique_output_dir("export_sb_data") obj_star = loader.load_by_patient(['ICb1299', '3021'], source='star', type='cell_culture', include_control=False) obj_salmon = loader.load_by_patient(['ICb1299', '3021'], source='salmon', type='cell_culture', include_control=False) # cluster plot tpm = filter.filter_by_cpm(obj_salmon.data, min_cpm=1, min_n_samples=4) batch_colours = common.COLOUR_BREWERS[len(obj_salmon.meta.batch.unique())] line_colours = common.COLOUR_BREWERS[2] cc = pd.DataFrame(line_colours[0], index=tpm.columns, columns=['Batch', 'Cell line']) aa, bb = obj_salmon.meta.batch.factorize() for i in range(aa.max()): cc.loc[aa == i, 'Batch'] = batch_colours[i] cc.loc[cc.index.str.contains('3021'), 'Cell line'] = line_colours[1] cg = clustering.dendrogram_with_colours( np.log2(tpm + eps), cc, ) cg['fig'].savefig(os.path.join(outdir, "dendrogram_pearson_log_tpm_all_genes.png"), dpi=200)