def __GO_enrich__(self): go_file = "go-basic.obo" if not os.path.exists(go_file): download_go_basic_obo() # Load gene ontologies obodag = GODag("go-basic.obo") # Read NCBI's gene2go. Store annotations in a list of namedtuples fin_gene2go = download_ncbi_associations() objanno = Gene2GoReader(fin_gene2go, taxids=[9606]) # Get namespace2association where: # namespace is: # BP: biological_process # MF: molecular_function # CC: cellular_component # association is a dict: # key: NCBI GeneID # value: A set of GO IDs associated with that gene ns2assoc = objanno.get_ns2assc() self.goeaobj = GOEnrichmentStudyNS( GeneID2nt_hum.keys(), # List of human protein-acoding genes ns2assoc, # geneID/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=['fdr_bh']) # default multipletest correction method
def run_go_enrichment(strain, genes_of_interest, significant=True, cutoff=0.05, use_parent_terms=True): # Load GO term association dictionary with open(os.path.join('data', 'go_association.pickle'), 'rb') as handle: go_association = pickle.load(handle) background_genes = get_genes( os.path.join('data', strain + '_all_genes.csv')) obo_go_fname = download_go_basic_obo() obo_dag = GODag('go-basic.obo') if strain == 'PA14': genes_of_interest = map_pa14_genes(genes_of_interest) background_genes = map_pa14_genes(background_genes) goea_obj = GOEnrichmentStudyNS(background_genes, go_association, obo_dag, propagate_counts=use_parent_terms, alpha=cutoff, methods=['fdr_bh']) goea_results = goea_obj.run_study(genes_of_interest) if significant is True: goea_results = [ result for result in goea_results if result.p_fdr_bh < cutoff ] enrichment_results = get_enrichment_results(goea_results) return [enrichment_results, goea_results]
def __init__( self, work_dir: str = '.', clean_work_dir: bool = False, organism: str = 'human', study_parameters: Dict[str, Union[int, float, str, List, Dict]] = { 'propagate_counts': False, 'alpha': 0.05, 'methods': ['fdr_bh'] } ) -> GOEngine: """A GOEngine that can be used for performing analysis using GOATOOLS Args: work_dir (str, optional): The path to a temp directory were intermediate-results and raw data will be downloaded/written to. Defaults to the current working directory. clean_work_dir (bool, optional): Whether or not to remove data written to the work directory at class termination, default to True. organism (str, optional): The organism . Defaults to 'human'. study_parameters (Dict[str,Union[int,float,str,List,Dict]], optional): A dict of parameters to control the base function, defaults to {'propagate_counts':False,'alpha':0.05, 'methods':['fdr_bh']} Returns: GOEngine: return a GO engine that can be used for performing GO enrichment analysis GOEnrichmentStudyNS """ print("Creating a GO Engine ...") if not os.path.exists(work_dir): raise ValueError( f"The provided work path: {work_dir} does not exist!!!") self.work_dir = work_dir if organism != 'human' and organism != 'mouse': raise ValueError( f"The provided organism: {organism} is not support, current engine mainly work with human and moues only" ) print(f"\t --> Downloading data ...") obo_fname = download_go_basic_obo( os.path.join(work_dir, 'go-basic.obo')) gene2go_fname = download_ncbi_associations( os.path.join(work_dir, 'gene2go')) ## parse the GO term print( f"\t --> parsing the data and intializing the base GOEA object...") obo_dag = GODag(obo_fname) if organism == 'human': self._goea_obj = GOEnrichmentStudyNS( gene2iden_human.keys(), Gene2GoReader(gene2go_fname, taxids=[9606]).get_ns2assc(), obo_dag, **study_parameters) else: self._goea_obj = GOEnrichmentStudyNS( gene2iden_human.keys(), Gene2GoReader(gene2go_fname, taxids=[10090]).get_ns2assc(), obo_dag, **study_parameters) self._clean_work_dir = clean_work_dir self._gene_ids = None return
def _get_objgoeans(pop, ns2assoc, godag): """Run gene ontology enrichment analysis (GOEA).""" return GOEnrichmentStudyNS(pop, ns2assoc, godag, propagate_counts=True, relationships=False, alpha=0.05, methods={'fdr_bh'})
def _init_objgoeans(self, pop, ns2assoc): """Run gene ontology enrichment analysis (GOEA).""" propagate_counts = not self.args.no_propagate_counts return GOEnrichmentStudyNS(pop, ns2assoc, self.godag, propagate_counts=propagate_counts, relationships=False, alpha=self.args.alpha, pvalcalc=self.args.pvalcalc, methods=self.methods)
def _get_goeaobj(methods=None): """Test GOEA with method, fdr.""" # REad GODag obo_fin = os.path.join(REPO, "go-basic.obo") obo_dag = get_godag(obo_fin, loading_bar=None) # Read association fin_assc = "{REPO}/tests/data/small_association".format(REPO=REPO) objanno = IdToGosReader(fin_assc, godag=obo_dag) ns2assc = objanno.get_ns2assc() popul_fin = "{REPO}/tests/data/small_population".format(REPO=REPO) popul_ids = [line.rstrip() for line in open(popul_fin)] goeaobj = GOEnrichmentStudyNS(popul_ids, ns2assc, obo_dag, methods=methods) return goeaobj
def gene_set_query(genes, fdr_threshold=0.10, return_header=False, species='mouse'): """ Runs a GO enrichment analysis query using goatools. The GO dataset here is for mouse, but it might apply to human as well. """ ns2assoc, ids_to_symbols, symbols_to_ids, genes_list = get_species_genes( species) goeaobj = GOEnrichmentStudyNS( genes_list, # List of mouse protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts=False, alpha=fdr_threshold, # default significance cut-off methods=['fdr_bh']) # defult multipletest correction method if species == 'mouse' or species == 'mus_musculus': genes = [x.capitalize() for x in genes] else: genes = [x.upper() for x in genes] gene_ids = [symbols_to_ids[x] for x in genes if x in symbols_to_ids] print('gene_ids:', gene_ids) results = goeaobj.run_study(gene_ids) results_sig = [r for r in results if r.p_fdr_bh < fdr_threshold] results_table = [] for r in results_sig: results_table.append([ r.goterm.id, r.goterm.name, r.p_fdr_bh, [ids_to_symbols[gene_id] for gene_id in r.study_items] ]) print(results_table) results_table.sort(key=lambda x: x[2]) if return_header: results_table = [['GO ID', 'Name', 'FDR', 'Overlapping Genes'] ] + results_table print('GO results_table:', results_table) return results_table
def GOEA(genes, objanno): """ returns go term enrichment Keyword arguments: genes -- list of genes objanno -- background dict performs GO term enrichment """ goeaobj = GOEnrichmentStudyNS( objanno.get_id2gos().keys(), # List of mouse protein-coding genes objanno.get_ns2assc(), # geneid/GO associations godag, # Ontologies propagate_counts=True, alpha=0.05, # default significance cut-off methods=['fdr_bh']) # defult multipletest correction method goea_quiet_all = goeaobj.run_study(genes, prt=None) goea_results = dict((el, []) for el in ontologies) for r in goea_quiet_all: goea_results[r.NS].append([r.GO, r.p_fdr_bh]) for ont in goea_results: goea_results[ont] = np.array(goea_results[ont]) goea_results[ont] = goea_results[ont][goea_results[ont][:, 0].argsort()] return goea_results
def _init_objgoeans(self, pop): """Run gene ontology enrichment analysis (GOEA).""" ns2assoc = self.objanno.get_ns2assc(**self._get_anno_kws()) ## BROAD rm_goids = self._get_remove_goids() rm_goids = False # BROAD return GOEnrichmentStudyNS( pop, ns2assoc, self.godag, propagate_counts=not self.args.no_propagate_counts, relationships=self.args.relationships, alpha=self.args.alpha, pvalcalc=self.args.pvalcalc, methods=self.methods, remove_goids=rm_goids)
def prep_goea(taxid=9606, prop_counts=True, alpha=0.05, method='fdr_bh', ref_list=None): ### DOWNLOAD AND LOAD ALL THE GENE STUFF for GOEA # download ontology from goatools.base import download_go_basic_obo obo_fname = download_go_basic_obo() # download associations from goatools.base import download_ncbi_associations fin_gene2go = download_ncbi_associations() # load ontology from goatools.obo_parser import GODag obodag = GODag("go-basic.obo") # load human gene ontology from goatools.anno.genetogo_reader import Gene2GoReader objanno = Gene2GoReader(fin_gene2go, taxids=[taxid ]) #9606 is taxonomy ID for h**o sapiens ns2assoc = objanno.get_ns2assc() for nspc, id2gos in ns2assoc.items(): print("{NS} {N:,} annotated human genes".format(NS=nspc, N=len(id2gos))) from goatools.goea.go_enrichment_ns import GOEnrichmentStudyNS #pop_ids = pd.read_csv('../data/df_human_geneinfo.csv',index_col=0)['GeneID'].to_list() df_genehumans = pd.read_csv('../data/df_human_geneinfo.csv', index_col=0) # if no reference list is given, default to all genes in ABHA if ref_list is None: ref_list = df_genehumans['GeneID'].to_list() goeaobj = GOEnrichmentStudyNS(ref_list, ns2assoc, obodag, propagate_counts=prop_counts, alpha=alpha, methods=[method]) # get symbol to ID translation dictionary to get overexpressed IDs symbol2id = dict( zip(df_genehumans['Symbol'].str.upper(), df_genehumans['GeneID'])) return goeaobj, symbol2id
def load_files(obo_fname, fin_gene2go): """function to load ontologies, associations and background gene set and then initialise a GOEA object""" # import the python module created in generate_background() # find specificy the current folder lcoation as the location of the module import sys sys.path.insert(1, ".") # import the module from genes_ncbi_3702_proteincoding import GENEID2NT as GeneID2nt_ara # load ontologies obodag = GODag(obo_fname) # load associations # Read NCBI's gene2go. Store Arabidopsis thaliana annotations in a list of named tuples objanno = Gene2GoReader(fin_gene2go, taxids=[3702]) # Get namespace2association where: # namespace is: # BP: biological_process # MF: molecular_function # CC: cellular_component # assocation is a dict: # key: NCBI GeneID # value: A set of GO IDs associated with that gene ns2assoc = objanno.get_ns2assc() for nspc, id2gos in ns2assoc.items(): print("{NS} {N:,} annotated Arabidopsis genes".format(NS=nspc, N=len(id2gos))) goeaobj = GOEnrichmentStudyNS( GeneID2nt_ara.keys( ), # List of filtered Arabidopsis protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=["fdr_bh"], ) # defult multipletest correction method return goeaobj, obodag, ns2assoc
def plot_go_enrichment(coef_df, auc_vals, pheno_dict, args, mode='abs'): obo_fl = os.path.join(args.go_dir, "go-basic.obo") download_go_basic_obo(obo_fl) obodag = GODag(obo_fl) assoc_fl = os.path.join(args.go_dir, "gene2go") download_ncbi_associations(assoc_fl) objanno = Gene2GoReader(assoc_fl, taxids=[9606]) ns2assoc = objanno.get_ns2assc() ncbi_map = {info.Symbol: ncbi_id for ncbi_id, info in GENEID2NT.items()} use_genes = set(coef_df.columns) & set(ncbi_map) bgrd_ids = [ncbi_map[gn] for gn in use_genes] goeaobj = GOEnrichmentStudyNS(bgrd_ids, ns2assoc, obodag, propagate_counts=False, alpha=0.05, methods=['fdr_bh']) plot_dict = dict() use_gos = set() coef_mat = coef_df.loc[:, [gene in use_genes for gene in coef_df.columns]] if mode == 'bayes': coef_means = coef_mat.groupby(level=0, axis=1).mean() coef_stds = coef_mat.groupby(level=0, axis=1).std() else: coef_mat = coef_mat.groupby(level=0, axis=1).mean() for mtype, coefs in coef_mat.iterrows(): if not isinstance(mtype, RandomType): if mode == 'abs': fgrd_ctf = coefs.abs().quantile(0.95) fgrd_genes = coefs.index[coefs.abs() > fgrd_ctf] use_clr = 3.17 elif mode == 'high': fgrd_ctf = coefs.quantile(0.95) fgrd_genes = coefs.index[coefs > fgrd_ctf] use_clr = 2.03 elif mode == 'low': fgrd_ctf = coefs.quantile(0.05) fgrd_genes = coefs.index[coefs < fgrd_ctf] use_clr = 1.03 elif mode == 'bayes': gene_scrs = coef_means.loc[mtype].abs() - coef_stds.loc[mtype] fgrd_genes = gene_scrs.index[gene_scrs > 0] use_clr = 3.17 else: raise ValueError( "Unrecognized `mode` argument <{}>!".format(mode)) fgrd_ids = [ncbi_map[gn] for gn in fgrd_genes] goea_out = goeaobj.run_study(fgrd_ids, prt=None) plot_dict[mtype] = { rs.name: np.log10(rs.p_fdr_bh) for rs in goea_out if rs.enrichment == 'e' and rs.p_fdr_bh < 0.05 } plot_df = pd.DataFrame(plot_dict, columns=plot_dict.keys()) if plot_df.shape[0] == 0: print("Could not find any enriched GO terms across {} " "subgroupings!".format(plot_df.shape[1])) return None fig, ax = plt.subplots(figsize=(4.7 + plot_df.shape[0] / 2.3, 2 + plot_df.shape[1] / 5.3)) if plot_df.shape[0] > 2: plot_df = plot_df.iloc[dendrogram(linkage(distance.pdist( plot_df.fillna(0.0), metric='cityblock'), method='centroid'), no_plot=True)['leaves']].transpose() else: plot_df = plot_df.transpose() xlabs = [rs_nm for rs_nm in plot_df.columns] ylabs = [ get_fancy_label(tuple(mtype.subtype_iter())[0][1]) for mtype in plot_df.index ] pval_cmap = sns.cubehelix_palette(start=use_clr, rot=0, dark=0, light=1, reverse=True, as_cmap=True) sns.heatmap(plot_df, cmap=pval_cmap, vmin=-5, vmax=0, linewidths=0.23, linecolor='0.73', xticklabels=xlabs, yticklabels=ylabs) ax.set_xticklabels(xlabs, size=15, ha='right', rotation=31) ax.set_yticklabels(ylabs, size=9, ha='right', rotation=0) ax.set_xlim((plot_df.shape[1] / -83, plot_df.shape[1] * 1.009)) ax.set_ylim((plot_df.shape[0] * 1.009, plot_df.shape[0] / -83)) plt.savefig(os.path.join( plot_dir, '__'.join([args.expr_source, args.cohort]), "{}_go-{}-enrichment_{}.svg".format(args.gene, mode, args.classif)), bbox_inches='tight', format='svg') plt.close()
obo_fname = download_go_basic_obo() fin_gene2go = download_ncbi_associations() if not download: obo_fname = '/d1/studies/singleCellTools/go-basic.obo' fin_gene2go = '/d1/studies/singleCellTools/gene2go' obodag = GODag("go-basic.obo") objanno = Gene2GoReader(fin_gene2go, taxids=[10090]) ns2assoc = objanno.get_ns2assc() for nspc, id2gos in ns2assoc.items(): print("{NS} {N:,} annotated mouse genes".format(NS=nspc, N=len(id2gos))) goeaobj = GOEnrichmentStudyNS( GeneID2nt_mus.keys(), # List of mouse protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts = False, alpha = 0.05, # default significance cut-off methods = ['fdr_bh']) # defult multipletest correction method def getEntrezIDs(genes): ID = [] IDs = [] nullGenes = [] for gene in genes: try: gene_info = geneIndex.loc[geneIndex.symbol==str(gene)] entrez_id = gene_info['entrez_id'].astype(int) eid = entrez_id.unique() if len(eid) == 0:
class Pose(object): def __init__(self, data_dir: str, device='cpu'): # load pretrained model self.model, self.name = self.__pretrained_model_construction__() self.model.load_state_dict( torch.load(data_dir + self.name + '-model.pt')) self.device = device self.__GO_enrich__() def __GO_enrich__(self): go_file = "go-basic.obo" if not os.path.exists(go_file): download_go_basic_obo() # Load gene ontologies obodag = GODag("go-basic.obo") # Read NCBI's gene2go. Store annotations in a list of namedtuples fin_gene2go = download_ncbi_associations() objanno = Gene2GoReader(fin_gene2go, taxids=[9606]) # Get namespace2association where: # namespace is: # BP: biological_process # MF: molecular_function # CC: cellular_component # association is a dict: # key: NCBI GeneID # value: A set of GO IDs associated with that gene ns2assoc = objanno.get_ns2assc() self.goeaobj = GOEnrichmentStudyNS( GeneID2nt_hum.keys(), # List of human protein-acoding genes ns2assoc, # geneID/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=['fdr_bh']) # default multipletest correction method def __pretrained_model_construction__(self): nhids_gcn = [64, 32, 32] prot_out_dim = sum(nhids_gcn) drug_dim = 128 pp = PP(gdata.n_prot, nhids_gcn) pd = PD(prot_out_dim, drug_dim, gdata.n_drug) mip = MultiInnerProductDecoder(drug_dim + pd.d_dim_feat, gdata.n_et) name = 'poly-' + str(nhids_gcn) + '-' + str(drug_dim) return Model(pp, pd, mip).to('cpu'), name def get_prediction_train(self, threshold=0.5): train_idx, train_et = remove_bidirection(gdata.train_idx, gdata.train_et) return self.predict(train_idx[0].tolist(), train_idx[1].tolist(), train_et.tolist(), threshold=threshold) def get_prediction_test(self, threshold=0.5): test_idx, test_et = remove_bidirection(gdata.test_idx, gdata.test_et) return self.predict(test_idx[0].tolist(), test_idx[1].tolist(), test_et.tolist(), threshold=threshold) def predict(self, drug1, drug2, side_effect, threshold=0.5): device = self.device data = gdata.to(device) model = self.model.to(device) model.eval() pp_static_edge_weights = torch.ones( (data.pp_index.shape[1])).to(device) pd_static_edge_weights = torch.ones( (data.pd_index.shape[1])).to(device) z = model.pp(data.p_feat, data.pp_index, pp_static_edge_weights) z0 = z.clone() z1 = z.clone() # prediction based on all infor z = model.pd(z, data.pd_index, pd_static_edge_weights) P = torch.sigmoid((z[drug1] * z[drug2] * model.mip.weight[side_effect]).sum(dim=1)).to('cpu') index_filter = P > threshold drug1 = torch.Tensor(drug1)[index_filter].numpy().astype(int).tolist() if not drug1: raise ValueError( "No Satisfied Edges." + "\n - Suggestion: reduce the threshold probability." + "Current probability threshold is {}. ".format(threshold) + "\n - Please use -h for help") drug2 = torch.Tensor(drug2)[index_filter].numpy().astype(int).tolist() side_effect = torch.Tensor(side_effect)[index_filter].numpy().astype( int).tolist() # prediction based on protein info and their interactions z0.data[:, 64:] *= 0 z0 = model.pd(z0, data.pd_index, pd_static_edge_weights) P0 = torch.sigmoid( (z0[drug1] * z0[drug2] * model.mip.weight[side_effect]).sum(dim=1)).to("cpu") ppiu_score = (P[index_filter] - P0) / P[index_filter] # prediction based on drug info only z1.data *= 0 z1 = model.pd(z1, data.pd_index, pd_static_edge_weights) P1 = torch.sigmoid( (z1[drug1] * z1[drug2] * model.mip.weight[side_effect]).sum(dim=1)).to("cpu") piu_score = (P[index_filter] - P1) / P[index_filter] # reture a query object query = PoseQuery(drug1, drug2, side_effect) query.set_pred_result(P[index_filter].tolist(), piu_score.tolist(), ppiu_score.tolist()) return query def explain_list(self, drug_list_1, drug_list_2, side_effect_list, regulization=2, if_auto_tuning=True, if_pred=True): if if_pred: query = self.predict(drug_list_1, drug_list_2, side_effect_list) else: query = PoseQuery(drug_list_1, drug_list_2, side_effect_list, regulization) return self.explain_query(query, if_auto_tuning=if_auto_tuning, regulization=query.regulization) def explain_query(self, query, if_auto_tuning=True, regulization=2): query.regulization = regulization pp_left_index, pp_left_weight, pd_left_index, pd_left_weight = self.__explain( query) if if_auto_tuning: while pp_left_index.shape[1] == 0: if query.regulization < 0.0001: print("Warning: auto tuning forced to stop.") break query.regulization /= 2 pp_left_index, pp_left_weight, pd_left_index, pd_left_weight = self.__explain( query) query.set_exp_result(pp_left_index, pp_left_weight, pd_left_index, pd_left_weight) goea_results_sig = self.enrich_go(pp_left_index) query.set_enrich_result(goea_results_sig) return query def enrich_go(self, pp_left_index): # -------------- Go Enrichment -------------- geneids_study = pp_left_index.flatten() # geneid2symbol.keys() geneids_study = [ int(gdata.prot_idx_to_id[idx].replace('GeneID', '')) for idx in geneids_study ] goea_results_all = self.goeaobj.run_study(geneids_study) goea_results_sig = [r for r in goea_results_all if r.p_fdr_bh < 0.05] return goea_results_sig def __explain(self, query): data = gdata model = self.model device = self.device drug_list_1, drug_list_2, side_effect_list, regulization = query.get_query( ) pre_mask = Pre_mask(data.pp_index.shape[1] // 2, data.pd_index.shape[1]).to(device) data = data.to(device) model = model.to(device) for gcn in self.model.pp.conv_list: gcn.cached = False self.model.pd.conv.cached = False self.model.eval() # pp_static_edge_weights = torch.ones((data.pp_index.shape[1])).to(device) # pd_static_edge_weights = torch.ones((data.pd_index.shape[1])).to(device) optimizer = torch.optim.Adam(pre_mask.parameters(), lr=0.01) fake_optimizer = torch.optim.Adam(model.parameters(), lr=0.01) # z = model.pp(data.p_feat, data.pp_index, pp_static_edge_weights) # z = model.pd(z, data.pd_index, pd_static_edge_weights) # # P = torch.sigmoid((z[drug1, :] * z[drug2, :] * model.mip.weight[side_effect, :]).sum()) # P = torch.sigmoid((z[drug_list_1] * z[drug_list_2] * model.mip.weight[side_effect_list]).sum(dim=1)) # if len(drug_list_1) < 5: # print(P.tolist()) tmp = 0.0 pre_mask.reset_parameters() for i in range(9999): model.train() pre_mask.desaturate() optimizer.zero_grad() fake_optimizer.zero_grad() # half_mask = torch.sigmoid(pre_mask.pp_weight) half_mask = torch.nn.Hardtanh(0, 1)(pre_mask.pp_weight) pp_mask = torch.cat([half_mask, half_mask]) pd_mask = torch.nn.Hardtanh(0, 1)(pre_mask.pd_weight) z = model.pp(data.p_feat, data.pp_index, pp_mask) # TODO: # z = model.pd(z, data.pd_index, pd_static_edge_weights) z = model.pd(z, data.pd_index, pd_mask) # TODO: # P = torch.sigmoid((z[drug1, :] * z[drug2, :] * model.mip.weight[side_effect, :]).sum()) P = torch.sigmoid((z[drug_list_1] * z[drug_list_2] * model.mip.weight[side_effect_list]).sum(dim=1)) EPS = 1e-7 # TODO: loss = torch.log(1 - P + EPS).sum() / regulization \ + 0.5 * (pp_mask * (2 - pp_mask)).sum() \ + (pd_mask * (2 - pd_mask)).sum() # loss = - torch.log(P) + 0.5 * (pp_mask * (2 - pp_mask)).sum() + (pd_mask * (2 - pd_mask)).sum() # TODO: loss.backward() optimizer.step() # print("Epoch:{}, loss:{}, prob:{}, pp_link_sum:{}, pd_link_sum:{}".format(i, loss.tolist(), P.tolist(), pp_mask.sum().tolist(), pd_mask.sum().tolist())) if i % 100 == 0: print( "Epoch:{:3d}, loss:{:0.2f}, prob:{:0.2f}, pp_link_sum:{:0.2f}, pd_link_sum:{:0.2f}" .format(i, loss.tolist(), P.mean().tolist(), pp_mask.sum().tolist(), pd_mask.sum().tolist())) # until no weight need to be updated --> no sum of weights changes if tmp == (pp_mask.sum().tolist(), pd_mask.sum().tolist()): break else: tmp = (pp_mask.sum().tolist(), pd_mask.sum().tolist()) pre_mask.saturate() pp_left_mask = (pp_mask > 0.2).detach().cpu().numpy() tmp = (data.pp_index[0, :] > data.pp_index[1, :]).detach().cpu().numpy() pp_left_mask = np.logical_and(pp_left_mask, tmp) pd_left_mask = (pd_mask > 0.2).detach().cpu().numpy() pp_left_index = data.pp_index[:, pp_left_mask].cpu().numpy() pd_left_index = data.pd_index[:, pd_left_mask].cpu().numpy() pp_left_weight = pp_mask[pp_left_mask].detach().cpu().numpy() pd_left_weight = pd_mask[pd_left_mask].detach().cpu().numpy() return pp_left_index, pp_left_weight, pd_left_index, pd_left_weight
def go_enrichment(gene_list, taxid=9606, background_chrom=None, background_genes=None, terms=None, list_study_genes=False, alpha=0.05): if type(gene_list) is pd.core.series.Series: gene_list = gene_list.tolist() if type(terms) is pd.core.series.Series: terms = terms.tolist() _assert_entrez_email() gene_list = list(gene_list) taxid = _tidy_taxid(taxid) ncbi_tsv = f'geneinfo_cache/{taxid}_protein_genes.txt' if not os.path.exists(ncbi_tsv): fetch_background_genes(taxid) with open(os.devnull, 'w') as null, redirect_stdout(null): obo_fname = download_and_move_go_basic_obo(prt=null) file_gene2go = download_ncbi_associations(prt=null) obodag = GODag("geneinfo_cache/go-basic.obo", optional_attrs=['relationship', 'def'], prt=null) # read NCBI's gene2go. Store annotations in a list of namedtuples objanno = Gene2GoReader(file_gene2go, taxids=[taxid]) # get associations for each branch of the GO DAG (BP, MF, CC) ns2assoc = objanno.get_ns2assc() # limit go dag to a sub graph including only specified terms and their children if terms is not None: sub_obo_name = 'geneinfo_cache/' + str( hash(''.join(sorted(terms)).encode())) + '.obo' wrsobo = WrSubObo(obo_fname, optional_attrs=['relationship', 'def']) wrsobo.wrobo(sub_obo_name, terms) obodag = GODag(sub_obo_name, optional_attrs=['relationship', 'def'], prt=null) # load background gene set of all genes background_genes_file = f'geneinfo_cache/{taxid}_protein_genes.txt' if not os.path.exists(background_genes_file): fetch_background_genes(taxid) # # load any custum subset if background_genes: if not all(type(x) is int for x in background_genes): if all(x.isnumeric() for x in background_genes): background_genes = list(map(str, background_genes)) else: background_genes = _cached_symbol2ncbi(background_genes, taxid=taxid) df = pd.read_csv(background_genes_file, sep='\t') no_suffix = os.path.splitext(background_genes_file)[0] background_genes_file = f'{no_suffix}_{hash("".join(map(str, sorted(background_genes))))}.txt' df.loc[df.GeneID.isin(background_genes)].to_csv( background_genes_file, sep='\t', index=False) # limit background gene set if background_chrom is not None: df = pd.read_csv(background_genes_file, sep='\t') background_genes_file = f'{os.path.splitext(background_genes_file)[0]}_{background_chrom}.txt' df.loc[df.chromosome == background_chrom].to_csv( background_genes_file, sep='\t', index=False) output_py = f'geneinfo_cache/{taxid}_background.py' ncbi_tsv_to_py(background_genes_file, output_py, prt=null) background_genes_name = output_py.replace('.py', '').replace('/', '.') background_genes = importlib.import_module(background_genes_name) importlib.reload(background_genes) GeneID2nt = background_genes.GENEID2NT if not all(type(x) is int for x in gene_list): gene_list = _cached_symbol2ncbi(gene_list, taxid=taxid) goeaobj = GOEnrichmentStudyNS( GeneID2nt, # List of mouse protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=['fdr_bh'], pvalcalc='fisher_scipy_stats') goea_results_all = goeaobj.run_study(gene_list) rows = [] columns = [ 'namespace', 'term_id', 'e/p', 'pval_uncorr', 'p_fdr_bh', 'ratio', 'bg_ratio', 'obj' ] if list_study_genes: columns.append('study_genes') for ntd in goea_results_all: ntd.__class__ = My_GOEnrichemntRecord # Hack. Changes __class__ of all instances... row = [ ntd.NS, ntd.GO, ntd.enrichment, ntd.p_uncorrected, ntd.p_fdr_bh, ntd.ratio_in_study[0] / ntd.ratio_in_study[1], ntd.ratio_in_pop[0] / ntd.ratio_in_pop[1], ntd ] if list_study_genes: row.append(_cached_ncbi2symbol(sorted(ntd.study_items))) rows.append(row) df = (pd.DataFrame().from_records(rows, columns=columns).sort_values( by=['p_fdr_bh', 'ratio']).reset_index(drop=True)) return df.loc[df.p_fdr_bh < alpha]
### to download a set of background population genes from NCBI. from genes_ncbi_10090_proteincoding import GENEID2NT as GeneID2nt_mus ### 3. INITIALIZE A GOEA object ### The GOEA object holds the Ontologies, Associations, and background. ### Numerous studies can then be run withough needing to re-load the above items. ### In this case, we only run one GOEA. from goatools.goea.go_enrichment_ns import GOEnrichmentStudyNS goeaobj = GOEnrichmentStudyNS( GeneID2nt_mus, # List of mouse protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts = False, alpha = 0.05, # default significance cut-off methods = ['fdr_bh']) # defult multipletest correction method ### 4. READ STUDY GENES # Data will be stored in this variable import os geneid2symbol = {} # Get xlsx filename where data is stored din_xlsx = r"C:\Users\krishna\Downloads\padj_converted.xlsx" ###excel file containing 3 columns: ### gene_symbols (our test data), their respective ESENMBL gene ids, and their p adj values (test_data) # Read data
def pullGOenrichment(inputFile, project): GeneID2nt_hum = genes_NCBI_9606_ProteinCoding.GENEID2NT obo_fname = download_go_basic_obo() fin_gene2go = download_ncbi_associations() obodag = GODag("go-basic.obo") # Read NCBI's gene2go. Store annotations in a list of namedtuples objanno = Gene2GoReader(fin_gene2go, taxids=[9606]) # Get namespace2association where: # namespace is: # BP: biological_process # MF: molecular_function # CC: cellular_component # assocation is a dict: # key: NCBI GeneID # value: A set of GO IDs associated with that gene ns2assoc = objanno.get_ns2assc() for nspc, id2gos in ns2assoc.items(): print("{NS} {N:,} annotated human genes".format(NS=nspc, N=len(id2gos))) print(len(GeneID2nt_hum)) goeaobj = GOEnrichmentStudyNS( GeneID2nt_hum.keys(), # List of human protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=['fdr_bh']) # defult multipletest correction method geneid2symbol = {} with open(inputFile, 'r') as infile: input_genes = csv.reader(infile) for line in input_genes: geneid = line[0] symbol = line[1] if geneid: geneid2symbol[int(geneid)] = symbol infile.close() geneids_study = geneid2symbol.keys() goea_results_all = goeaobj.run_study(geneids_study) goea_results_sig = [r for r in goea_results_all if r.p_fdr_bh < 0.05] import collections as cx ctr = cx.Counter([r.NS for r in goea_results_sig]) print('Significant results[{TOTAL}] = {BP} BP + {MF} MF + {CC} CC'.format( TOTAL=len(goea_results_sig), BP=ctr['BP'], # biological_process MF=ctr['MF'], # molecular_function CC=ctr['CC'])) # cellular_component goeaobj.wr_xlsx("Data/go_enrichment" + project + ".csv", goea_results_sig) goeaobj.wr_txt("Data/go_enrichment" + project + ".txt", goea_results_sig)
def keep_only_go(x): golist = x.split(';') cleanlist = {i.split('#')[-1] for i in golist} return cleanlist dfA['BP'] = dfA['BP'].apply(keep_only_go) dfA['UniProtKB/Swiss-Prot ID'] = dfA.index.map(dfQ.reset_index().set_index('PANTHER ID')['UniProtKB/Swiss-Prot ID'].dropna().to_dict()) dfA['Gene stable ID'] = dfA.index.map(dfQ.reset_index().set_index('PANTHER ID')['Gene stable ID'].dropna().to_dict()) ns2assoc_combined = dfA.set_index('UniProtKB/Swiss-Prot ID').loc[:, ['BP']].to_dict() """ # Gene Ontology Enrichment Analysis (GOEA) goea = GOEnrichmentStudyNS(pop=pop, ns2assoc=ns2assoc_combined, godag=godag, propagate_counts=True, alpha=0.05, methods=['fdr_bh']) # Load PCA rPCAFile = '../../04-network/results/pca/{celltype:s}/{layer:s}/pca-{celltype:s}-{network:s}-{threshold:s}-{layer:s}-dim.csv.gz'.format(celltype=celltype, network=network, threshold=threshold_str, layer=layer) rDiAnFile = '../../04-network/results/pca/{celltype:s}/{layer:s}/pca-{celltype:s}-{network:s}-{threshold:s}-{layer:s}-dian.csv.gz'.format(celltype=celltype, network=network, threshold=threshold_str, layer=layer) rEntFile = '../../04-network/results/pca/{celltype:s}/{layer:s}/pca-{celltype:s}-{network:s}-{threshold:s}-{layer:s}-entropy.csv.gz'.format(celltype=celltype, network=network, threshold=threshold_str, layer=layer) # wCSVFile = 'results/goea/{celltype:s}/goea-{celltype:s}-{network:s}-{threshold:s}-{layer:s}.csv.gz'.format(celltype=celltype, network=network, threshold=threshold_str, layer=layer) df_pca = pd.read_csv(rPCAFile, index_col=0) df_dian = pd.read_csv(rDiAnFile, index_col=0) df_ent = pd.read_csv(rEntFile, index_col=0) ldfS = [] for module in modules:
# assocation is a dict: # key: NCBI GeneID # value: A set of GO IDs associated with that gene ns2assoc = objanno.get_ns2assc() for nspc, id2gos in ns2assoc.items(): print("{NS} {N:,} annotated human genes".format(NS=nspc, N=len(id2gos))) print() print(len(GeneID2nt_homo)) print() goeaobj = GOEnrichmentStudyNS( GeneID2nt_homo.keys(), # List of mouse protein-coding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=['fdr_bh']) # defult multipletest correction method symbols = np.zeros(len(GeneID2nt_homo.keys()), dtype='U100') geneids = np.zeros(len(GeneID2nt_homo.keys()), dtype=int) #Creating a lookup table to convert the gene symbols to the gene ids needed for the gene enrichment analysis for idx, key in enumerate(GeneID2nt_homo.keys()): symbols[idx] = GeneID2nt_homo[key].Symbol geneids[idx] = GeneID2nt_homo[key].GeneID boolean_symbol = np.isin(symbols, CDK1_gene_list) matches_idx = np.where(boolean_symbol)[0]
# select genes significantly differentially expressed according to BH FDR of sleuth fdr_level_gene = float(snakemake.params.gene_fdr) sig_genes = all_genes[all_genes['qval']<fdr_level_gene] # initialize GOEA object fdr_level_go_term = float(snakemake.params.go_term_fdr) goeaobj = GOEnrichmentStudyNS( # list of 'population' of genes looked at in total pop = all_genes['ens_gene'].tolist(), # geneid -> GO ID mapping ns2assoc = ns2assoc, # ontology DAG godag = obodag, propagate_counts = False, # multiple testing correction method (fdr_bh is false discovery rate control with Benjamini-Hochberg) methods = ['fdr_bh'], # significance cutoff for method named above alpha = fdr_level_go_term ) goea_results_all = goeaobj.run_study(sig_genes['ens_gene'].tolist()) # write results to text file goeaobj.wr_tsv(snakemake.output.enrichment, goea_results_all) # plot results
# namespace is: # BP: biological_process # MF: molecular_function # CC: cellular_component # assocation is a dict: # key: NCBI GeneID # value: A set of GO IDs associated with that gene ns2assoc = objanno.get_ns2assc() for nspc, id2gos in ns2assoc.items(): print("{NS} {N:,} annotated human genes".format(NS=nspc, N=len(id2gos))) goeaobj = GOEnrichmentStudyNS( GeneID2nt_hum.keys(), # List of human protein-acoding genes ns2assoc, # geneid/GO associations obodag, # Ontologies propagate_counts=False, alpha=0.05, # default significance cut-off methods=['fdr_bh']) # defult multipletest correction method # 'p_' means "pvalue". 'fdr_bh' is the multipletest method we are currently using. geneids_study = pp_idx.flatten() # geneid2symbol.keys() geneids_study = [int(data.prot_idx_to_id[idx].replace('GeneID', '')) for idx in geneids_study] # ############################################################################## goea_results_all = goeaobj.run_study(geneids_study) goea_results_sig = [r for r in goea_results_all if r.p_fdr_bh < 0.05] # ############################################################################## n_sig = len(goea_results_sig)
class GOEngine: def __init__( self, work_dir: str = '.', clean_work_dir: bool = False, organism: str = 'human', study_parameters: Dict[str, Union[int, float, str, List, Dict]] = { 'propagate_counts': False, 'alpha': 0.05, 'methods': ['fdr_bh'] } ) -> GOEngine: """A GOEngine that can be used for performing analysis using GOATOOLS Args: work_dir (str, optional): The path to a temp directory were intermediate-results and raw data will be downloaded/written to. Defaults to the current working directory. clean_work_dir (bool, optional): Whether or not to remove data written to the work directory at class termination, default to True. organism (str, optional): The organism . Defaults to 'human'. study_parameters (Dict[str,Union[int,float,str,List,Dict]], optional): A dict of parameters to control the base function, defaults to {'propagate_counts':False,'alpha':0.05, 'methods':['fdr_bh']} Returns: GOEngine: return a GO engine that can be used for performing GO enrichment analysis GOEnrichmentStudyNS """ print("Creating a GO Engine ...") if not os.path.exists(work_dir): raise ValueError( f"The provided work path: {work_dir} does not exist!!!") self.work_dir = work_dir if organism != 'human' and organism != 'mouse': raise ValueError( f"The provided organism: {organism} is not support, current engine mainly work with human and moues only" ) print(f"\t --> Downloading data ...") obo_fname = download_go_basic_obo( os.path.join(work_dir, 'go-basic.obo')) gene2go_fname = download_ncbi_associations( os.path.join(work_dir, 'gene2go')) ## parse the GO term print( f"\t --> parsing the data and intializing the base GOEA object...") obo_dag = GODag(obo_fname) if organism == 'human': self._goea_obj = GOEnrichmentStudyNS( gene2iden_human.keys(), Gene2GoReader(gene2go_fname, taxids=[9606]).get_ns2assc(), obo_dag, **study_parameters) else: self._goea_obj = GOEnrichmentStudyNS( gene2iden_human.keys(), Gene2GoReader(gene2go_fname, taxids=[10090]).get_ns2assc(), obo_dag, **study_parameters) self._clean_work_dir = clean_work_dir self._gene_ids = None return def load_data(self, exp: Experiment, num_proteins: int = -1) -> None: """Load the data to the Engine, so GOEA can be conducted Args: exp (Experiment): An Experimental object to extract uniprot ids num_proteins (int, optional): The number of proteins to be included in the analysis. Defaults -1 to which mean use all proteins,\ otherwise it uses the number of proteins provided by the user. note that the function is sorted by number of peptides per protein,\ that is the first 10 protein means, getting the top 10 protein with most peptides. Raises: ValueError: if the function called while data being already associated with the engine from a previous call """ if self._gene_ids is not None: raise ValueError( f"There some data still in the engine, the first 10 genes are: {','.join(self._gene_ids[:10])}\ clean your engine from previous data using the function, clean_engine and try again." ) print( f"Getting the number of peptide per protein ..., started at: {time.ctime()}" ) num_protein_per_peptides = exp.get_peptides_per_protein() if num_proteins == -1: list_proteins = num_protein_per_peptides.iloc[:, 0].to_list() else: list_proteins = num_protein_per_peptides.iloc[:, 0].to_list( )[:num_proteins] print( f"Map uniprot to Entrez gene ids ..., starting at: {time.ctime()}") self._gene_ids = [ int(gene_id) for gene_id in map_from_uniprot_to_Entrez_Gene( list_proteins).iloc[:, 1].to_list() ] print(f"{len(self._gene_ids)} Genes have been correctly loaded") return def run_analysis( self, quite: bool = False, only_signifcant: bool = True, significance_level: float = 0.05, get_list_term: bool = False ) -> Union[pd.DataFrame, List[GOEnrichmentRecord]]: if quite: goea_results = self._goea_obj.run_study(self._gene_ids, prt=None) else: goea_results = self._goea_obj.run_study(self._gene_ids) if only_signifcant: goea_results = [ res for res in goea_results if res.p_fdr_bh < significance_level ] if get_list_term: return goea_results else: self._goea_obj.wr_tsv(os.path.join(self.work_dir, 'temp_file.tsv'), goea_results) results_df = pd.read_csv(os.path.join(self.work_dir, 'temp_file.tsv'), sep='\t') os.system(f"rm -f {os.path.join(self.work_dir,'temp_file.tsv')}") return results_df def clean_engine(self) -> None: """Remove Current list of gene ids associated with the engine """ self._gene_ids = None return def __del__(self) -> None: """class destructor, clean work directory if clean_work_dir is set to True """ if self.clean_work_dir: os.system(f"rm -f {self.work_dir}/*") return