def gene_to_ens(genes):
"""
Convert gene(s) found in the annotation to Ensembl gene IDs
:param genes: One or more gene names
:return: pd.Series indexed by genes containing corresponding ENS IDs where available
"""
if isinstance(genes, str):
genes = [genes]
df = reference_genomes.gene_symbol_to_ensembl(genes)
not_found = df.index[df.isna()]
for nf in not_found:
if nf in MANUALLY_CURATED:
df.loc[nf] = MANUALLY_CURATED[nf]
for g in UNRELIABLE:
if g in df:
df.loc[g] = None
return df
def tabulate_de_counts_by_direction(de_res, pids=consts.PIDS, **gene_lists):
"""
Given the lists of genes associated with the hypo and hyper group, tabulate the numbers of matching genes in
the supplied DE results for each patient. Split counts by DE logFC direction.
:param de_res:
:param genes_hypo:
:param genes_hyper:
:param pids:
:return: Raw counts table, Table expressing coutns as a % of the total in that direction
"""
cols = reduce(
lambda x, y: x + y,
[["%s up" % k, "%s down" % k] for k in gene_lists]
)
# table of DE counts (in DMR-linked context)
de_count_table = pd.DataFrame(
0,
index=pids,
columns=cols + ['Total up', 'Total down']
)
for pid in pids:
de_count_table.loc[pid, 'Total up'] = (de_res[pid]['logFC'] > 0).sum()
de_count_table.loc[pid, 'Total down'] = (de_res[pid]['logFC'] < 0).sum()
for k, g_arr in gene_lists.items():
ix = de_res[pid].index.intersection(reference_genomes.gene_symbol_to_ensembl(g_arr).dropna().values)
de_count_table.loc[pid, '%s up' % k] = (de_res[pid].loc[ix, 'logFC'] > 0).sum()
de_count_table.loc[pid, '%s down' % k] = (de_res[pid].loc[ix, 'logFC'] < 0).sum()
# express this as a pct of the total up/down
de_count_table_pct = pd.DataFrame(
index=pids,
columns=cols
)
for k in gene_lists:
for k2 in ['up', 'down']:
de_count_table_pct.loc[pids, "%s %s" % (k, k2)] = \
de_count_table.loc[pids, "%s %s" % (k, k2)] / de_count_table.loc[pids, 'Total %s' % k2].astype(float) * 100.
return de_count_table, de_count_table_pct
median_count = dat_n.median(axis=1).sort_values()
keep_idx = median_count.loc[median_count != 0].index
dat = dat.loc[keep_idx]
dat_n = dat_n.loc[keep_idx]
median_count = median_count.loc[keep_idx]
# remove any genes that are (mostly) absent in NSC
nsc_missing = dat.loc[:, ref] < 10.
dat = dat.loc[~nsc_missing]
dat_n = dat_n.loc[~nsc_missing]
median_count = median_count.loc[~nsc_missing]
# levels of HKG across GBM and GIBCO
hkg = ['ATP5B', 'GAPDH', 'ACTB']
hkg_ens = gene_symbol_to_ensembl(hkg)
label_symbols = hkg + ['BMI1']
label_ens = gene_symbol_to_ensembl(label_symbols)
hkg_dat = dat_n.loc[hkg_ens, sorted(dat_n.columns)]
hkg_dat.index = pd.Index(hkg, name='')
hkg_dat_rel = hkg_dat.divide(hkg_dat.loc[:, ref], axis=0)
ax = hkg_dat_rel.transpose().plot.bar()
ax.set_ylim([0, 3.4])
plt.tight_layout()
ax.figure.savefig(os.path.join(outdir, 'housekeeping_levels.png'), dpi=200)
# identifying stable HKG
ranked_count = pd.Series(rankdata(median_count, method='ordinal'),
index=median_count.index)
),
axis=0)
data = data.loc[data.index.str.contains('ENSG')]
# remove rRNA
data = data.loc[~data.index.isin(rrna_ensg)]
# remove MT RNA
data = data.loc[~data.index.isin(mt_ensg)]
# normalise by read counts: per million
data_n = data.divide(data.sum(axis=0), axis=1)
# extract genes of interest
genes = reference_genomes.gene_symbol_to_ensembl(['SLC1A3', 'PDGFRA'])
this_fpkm = data_n.loc[genes] * 1e6
this_fpkm.index = genes.index
# normalise by gene length: per million per kilobase
for g in genes.index:
this_fpkm.loc[g] = this_fpkm.loc[g] / gene_lengths[g] * 1e3
ax = this_fpkm.transpose().plot.bar()
ax.set_ylabel('FPKM')
ax.figure.tight_layout()
ax.figure.savefig(os.path.join(OUTDIR, 'all_markers_fpkm.png'), dpi=200)
# now just our own samples and the reference
this_fpkm_lim = this_fpkm.loc[:, this_fpkm.columns.str.contains('NSC')]
# for plotting
groups = [(pid, dat_s1.columns[dat_s1.columns.str.contains(pid)])
for pid in pids]
for name, p_arr in poi.items():
this_ipa = dict([(pid, ipa_de_res[pid].reindex(list(p_arr)))
for pid in pids])
# get all genes involved
union_genes = set()
for v in this_ipa.values():
for arr in v.dropna().genes.str.split(',').values:
union_genes.update(arr)
union_genes_ens = reference_genomes.gene_symbol_to_ensembl(union_genes)
union_genes_comp = union_genes_ens.reset_index().set_index(
'Ensembl Gene ID').squeeze()
# plot TPM
this_cpm = cpm.loc[union_genes_ens]
# sort by mean logFC across patients
this_de = de_res_wide.loc[union_genes_ens]
this_logfc = this_de.loc[:, this_de.columns.str.contains('logFC')]
this_logfc.columns = pids
sort_ix = (this_logfc.sum(axis=1) /
float(len(pids))).sort_values(ascending=False).index
this_logfc = this_logfc.loc[sort_ix]
this_cpm = this_cpm.loc[sort_ix]
method='single')
clustering.dendrogram_with_colours(data,
col_colours,
legend_labels=legend_labels,
metric='correlation',
method='average')
clustering.dendrogram_with_colours(data,
col_colours,
legend_labels=legend_labels,
metric='correlation',
method='single')
# concatenate meta and data for plotting
# variables in columns
ens_pdgfra = reference_genomes.gene_symbol_to_ensembl('PDGFRA')
ens_nf1 = reference_genomes.gene_symbol_to_ensembl('NF1')
ens_cdkn2a = reference_genomes.gene_symbol_to_ensembl('CDKN2A')
aa = pd.concat((datan.transpose(), meta), axis=1)
ax = sns.boxplot(y=ens_pdgfra, x='subgroup', data=aa)
ax.set_xlabel('Methylation subgroup')
ax.set_ylabel('PDGFRa (proportion of reads)')
plt.setp(ax.xaxis.get_ticklabels(), rotation=90)
plt.tight_layout()
ax.figure.savefig(os.path.join(outdir, "pdgfra_boxplot.png"), dpi=200)
ax.figure.savefig(os.path.join(outdir, "pdgfra_boxplot.pdf"))
ax = sns.boxplot(y=ens_nf1, x='subgroup', data=aa)
ax.set_xlabel('Methylation subgroup')
obj61794.meta,
objwtchg_all.meta,
objpollard.meta
), axis=0)
data = data.loc[data.index.str.contains('ENSG')]
# compare with qPCR: comparing markers in the NSC samples and paired astrocytes
astro_markers1 = [
'S100B',
'CD44',
'ALDH1L1',
'NFIA',
]
data_markers = data.loc[
reference_genomes.gene_symbol_to_ensembl(astro_markers1),
data.columns.str.contains('DURA')
]
data_markers.index = astro_markers1
series = [data_markers.iloc[i] for i in range(data_markers.shape[0])]
colours = [
'#0000FF',
'#FF0000',
'#00C000',
'#AD07E3',
]
# plot absolute counts
fig = plt.figure(figsize=(8, 7.5))
ax = fig.add_subplot(111)
# 'encode_roadmap/ENCSR291IZK': {'batch': 'ENCODE Ecker', 'strandedness': 'r'},
# 'encode_roadmap/ENCSR572EET': {'batch': 'ENCODE Ecker', 'strandedness': 'r'},
# 'encode_roadmap/ENCSR977XUX': {'batch': 'ENCODE Ecker', 'strandedness': 'r'},
}
to_aggr_nsc = [
(r'H9_NSC_[12]', 'H9 NSC'),
# (r'Pollard NSC [12]', 'Fetal NSC'),
]
# Ruiz 9 gene signature - should distinguish ESC and iPSC
gene_sign = [
'PTPRT', 'TMEM132C', 'TMEM132D', 'TCERG1L', 'DPP6', 'FAM19A5',
'RBFOX1', 'CSMD1', 'C22orf34'
]
gene_sign_ens = reference_genomes.gene_symbol_to_ensembl(gene_sign)
load_kwds = {'source': source, 'alignment_subdir': SetMe}
if source == 'salmon':
units = 'tpm'
load_kwds['units'] = 'tpm'
if source == 'star':
# set strandedness as a cue to import for each
load_kwds['strandedness'] = SetMe
# restrict samples manually to avoid changes going forwards
our_samples = consts.S1_RNASEQ_SAMPLES_INSC + consts.S1_RNASEQ_SAMPLES_IPSC + consts.S1_RNASEQ_SAMPLES_FB + [
'GIBCO_NSC_P4'
]
# our data (everything)
from utils import output
from matplotlib import pyplot as plt
import seaborn as sns
import pandas as pd
import numpy as np
from rnaseq import loader
from scripts.hgic_final import consts
from hgic_consts import NH_ID_TO_PATIENT_ID_MAP
from utils import reference_genomes
if __name__ == '__main__':
outdir = output.unique_output_dir()
eps = 0.1
seps = Decimal('0.1')
gois = ['PTGER4']
ens_gois = reference_genomes.gene_symbol_to_ensembl(gois)
# pids = consts.PIDS
pids = ['018']
meta_fn = os.path.join(DATA_DIR, 'rnaseq', 'tcga_gbm',
'primary_tumour/htseq-count_fpkm/sources.csv')
dat_fn = os.path.join(DATA_DIR, 'rnaseq', 'tcga_gbm',
'primary_tumour/htseq-count_fpkm/fpkm.csv')
tcga_meta = pd.read_csv(meta_fn, header=0, index_col=0)
tcga_dat = pd.read_csv(dat_fn, header=0, index_col=0)
# filter: primary GBM only
ix = (tcga_meta.idh1_status == 'WT')
tcga_meta = tcga_meta[ix]
tcga_dat = tcga_dat[tcga_meta.index]
tax_id = 9606
if isinstance(cell_types, str):
cell_types = [cell_types]
if unit == 'tpm':
dat = rnaseq_data.load_salmon_by_patient_id(pids,
include_control=False,
type=type)
dat = general.ensembl_transcript_quant_to_gene(dat, tax_id=tax_id)
elif unit == 'cpm':
obj = rnaseq_data.load_by_patient(pids,
type=type,
source='star',
annotate_by='Ensembl Gene ID',
include_control=False)
dat = obj.data
dat = dat.divide(dat.sum(axis=0), axis=1) * 1e6
else:
raise NotImplementedError()
if cell_types is not None:
idx = reduce(
lambda x, y: x | y,
[dat.columns.str.contains(t) for t in cell_types],
)
dat = dat.loc[:, idx]
lookup = reference_genomes.gene_symbol_to_ensembl(gois, tax_id=tax_id)
vals = dat.loc[lookup]
from plotting import common
from rnaseq import loader
from scripts.hgic_final import consts, two_strategies_grouped_dispersion as tsgd
from settings import HGIC_LOCAL_DIR, INTERMEDIATE_DIR
from stats import nht
from utils import output, setops, reference_genomes
if __name__ == "__main__":
outdir = output.unique_output_dir()
pids = consts.PIDS
DE_LOAD_DIR = os.path.join(INTERMEDIATE_DIR, 'de')
eps = .1 # offset for log transform
target_gene = 'CD274'
target_ens = reference_genomes.gene_symbol_to_ensembl(target_gene)
# load Salmon data
obj_cc = loader.load_by_patient(pids, source='salmon')
ix = obj_cc.meta.index.isin(consts.S1_RNASEQ_SAMPLES)
obj_cc.filter_samples(ix)
# add PID to cell culture metadata
obj_cc.meta.insert(
0, 'pid',
obj_cc.meta.index.str.replace(r'(GBM|DURA)(?P<pid>[0-9]{3}).*',
'\g<pid>'))
obj_ff = loader.load_by_patient(pids, source='salmon', type='ffpe')
obj_ff.filter_by_sample_name(consts.FFPE_RNASEQ_SAMPLES)
# is there much overlap in the gene sets between the two groups?
fig = plt.figure(figsize=(5, 3))
ax = fig.add_subplot(111)
venn.venn_diagram(genes_from_dmr_groups['Hyper'],
genes_from_dmr_groups['Hypo'],
set_labels=['Hyper', 'Hypo'],
ax=ax)
fig.tight_layout()
fig.savefig(os.path.join(outdir, "genes_from_dmr_groups_venn.png"),
dpi=200)
# no, but if we look at the intersection genes, are they in different directions (DE) between the two groups?
groups_inv = dictionary.complement_dictionary_of_iterables(groups,
squeeze=True)
in_both = setops.reduce_intersection(*genes_from_dmr_groups.values())
in_both_ens = reference_genomes.gene_symbol_to_ensembl(in_both)
# some of these will have no DE results
tmp = {}
for pid in pids:
tmp[pid] = de_res_s1[pid].reindex(in_both_ens).logFC.dropna()
in_both_ens = in_both_ens[in_both_ens.isin(pd.DataFrame(tmp).index)]
fig = plt.figure(figsize=(10.5, 4.))
ax = fig.add_subplot(111)
for pid in pids:
this_grp = groups_inv[pid].lower()
this_de = de_res_s1[pid].reindex(in_both_ens).logFC
x = np.arange(
len(in_both_ens)) + 0.1 * (-1 if this_grp == 'hypo' else 1)
ax.scatter(x,
def scatter_plot(pid,
de_res_s1,
de_res_s1_full,
de_res_s2,
ipa_df,
pw_colours,
ms=30,
s1_colour='darkblue',
s2_colour='lightskyblue',
external_ref_labels=('GIBCO', 'H9')):
external_ref_labels = list(external_ref_labels)
dat_all = pd.DataFrame.from_dict({
'logFC': de_res_s1_full[pid].logFC,
'pFDR': -np.log10(de_res_s1_full[pid].FDR)
})
# patient specific
vs, vc = setops.venn_from_arrays(*[de_res_s1[p].index for p in pids])
set_ps = list(setops.binary_combinations_sum_eq(len(pids), 1))
# remember that sets are defined in reverse!
set_ps = dict([(pids[i], vs[set_ps[(-1 - i)]]) for i in range(len(pids))])
dat_patient_specific = dat_all.loc[set_ps[pid]]
# syngeneic only
the_idx = [de_res_s2[(pid, k)].index for k in [pid] + external_ref_labels]
vs, vc = setops.venn_from_arrays(*the_idx)
dat_syngen_only = dat_all.loc[vs['100']]
# pathway(s)
in_pathways = {}
pw_dat = {}
for pw in pw_colours:
the_genes = ipa_df.loc[pw].genes.split(',')
the_ens_id = reference_genomes.gene_symbol_to_ensembl(
the_genes).dropna().tolist()
# resolve '/' weirdness
for g in the_genes:
if '/' in g:
g_arr = g.split('/')
e_arr = reference_genomes.gene_symbol_to_ensembl(
g_arr).dropna().tolist()
the_ens_id += e_arr
# drop any genes not found in the data
in_pathways[pw] = dat_all.index.intersection(the_ens_id)
pw_dat[pw] = dat_all.loc[in_pathways[pw]]
marker_full = markers.MarkerStyle(marker='o', fillstyle='full')
marker_s1 = markers.MarkerStyle(marker='o', fillstyle='left')
marker_s2 = markers.MarkerStyle(marker='o', fillstyle='right')
fig = plt.figure(figsize=(7.1, 6.3))
ax = fig.add_subplot(111)
ax.scatter(dat_all['logFC'],
dat_all['pFDR'],
c='gray',
s=ms,
zorder=1,
alpha=0.3,
label='All DE')
# S1
ax.scatter(
dat_patient_specific.loc[:, 'logFC'],
dat_patient_specific.loc[:, 'pFDR'],
marker=marker_s1,
c=s1_colour,
s=ms,
zorder=3, # in front as there are fewer
label='Patient specific')
ax.scatter(
dat_patient_specific.loc[:, 'logFC'],
dat_patient_specific.loc[:, 'pFDR'],
marker=marker_full,
edgecolor='k',
linewidths=.5,
c='none',
s=ms,
zorder=3, # in front as there are fewer
label=None,
)
# S2
ax.scatter(
dat_syngen_only['logFC'],
dat_syngen_only['pFDR'],
marker=marker_s2,
c=s2_colour,
s=ms,
zorder=2, # behind S1 as there are more
label='Syngeneic only')
ax.scatter(
dat_syngen_only['logFC'],
dat_syngen_only['pFDR'],
marker=marker_full,
edgecolor='k',
linewidths=.5,
c='none',
s=ms,
zorder=2, # behind S1 as there are more
label=None,
)
for pw in pw_colours:
ax.scatter(pw_dat[pw]['logFC'],
pw_dat[pw]['pFDR'],
c='none',
edgecolor=pw_colours[pw],
linewidths=1.5,
s=ms + 2,
zorder=4,
label=pw)
ax.legend(loc='upper right', frameon=True, facecolor='w', framealpha=0.7)
ax.set_xlabel('logFC')
ax.set_ylabel('-log10(FDR)')
fig.tight_layout()
return fig, ax