def _get_process_allelic_variants(self, entry, g): gu = GraphUtils(curie_map.get()) geno = Genotype(g) du = DipperUtil() if entry is not None: publist = {} # to hold the entry-specific publication mentions for the allelic variants entry_num = entry['mimNumber'] # process the ref list just to get the pmids ref_to_pmid = self._get_pubs(entry, g) if 'allelicVariantList' in entry: allelicVariantList = entry['allelicVariantList'] for al in allelicVariantList: al_num = al['allelicVariant']['number'] al_id = 'OMIM:'+str(entry_num)+'.'+str(al_num).zfill(4) al_label = None al_description = None if al['allelicVariant']['status'] == 'live': publist[al_id] = set() if 'mutations' in al['allelicVariant']: al_label = al['allelicVariant']['mutations'] if 'text' in al['allelicVariant']: al_description = al['allelicVariant']['text'] m = re.findall('\{(\d+)\:', al_description) publist[al_id] = set(m) geno.addAllele(al_id, al_label, geno.genoparts['variant_locus'], al_description) geno.addAlleleOfGene(al_id, 'OMIM:'+str(entry_num), geno.object_properties['is_sequence_variant_instance_of']) for r in publist[al_id]: pmid = ref_to_pmid[int(r)] gu.addTriple(g, pmid, gu.object_properties['is_about'], al_id) # look up the pubmed id in the list of references if 'dbSnps' in al['allelicVariant']: dbsnp_ids = re.split(',', al['allelicVariant']['dbSnps']) for dnum in dbsnp_ids: did = 'dbSNP:'+dnum.strip() gu.addIndividualToGraph(g, did, None) gu.addEquivalentClass(g, al_id, did) if 'clinvarAccessions' in al['allelicVariant']: # clinvarAccessions triple semicolon delimited, each lik eRCV000020059;;1 rcv_ids = re.split(';;;', al['allelicVariant']['clinvarAccessions']) rcv_ids = [(re.match('(RCV\d+)\;\;', r)).group(1) for r in rcv_ids] for rnum in rcv_ids: rid = 'ClinVar:'+rnum gu.addXref(g, al_id, rid) gu.addPage(g, al_id, "http://omim.org/entry/"+str(entry_num)+"#"+str(al_num).zfill(4)) elif re.search('moved', al['allelicVariant']['status']): # for both 'moved' and 'removed' moved_ids = None if 'movedTo' in al['allelicVariant']: moved_id = 'OMIM:'+al['allelicVariant']['movedTo'] moved_ids = [moved_id] gu.addDeprecatedIndividual(g, al_id, moved_ids) else: logger.error('Uncaught alleleic variant status %s', al['allelicVariant']['status']) # end loop allelicVariantList return
def _get_gene_history(self, limit): """ Loops through the gene_history file and adds the old gene ids as deprecated classes, where the new gene id is the replacement for it. The old gene symbol is added as a synonym to the gene. :param limit: :return: """ gu = GraphUtils(curie_map.get()) if self.testMode: g = self.testgraph else: g = self.graph logger.info("Processing Gene records") line_counter = 0 myfile = '/'.join((self.rawdir, self.files['gene_history']['file'])) logger.info("FILE: %s", myfile) with gzip.open(myfile, 'rb') as f: for line in f: # skip comments line = line.decode().strip() if re.match(r'^#', line): continue (tax_num, gene_num, discontinued_num, discontinued_symbol, discontinued_date) = line.split('\t') # set filter=None in init if you don't want to have a filter # if self.filter is not None: # if ((self.filter == 'taxids' and \ # (int(tax_num) not in self.tax_ids)) # or (self.filter == 'geneids' and \ # (int(gene_num) not in self.gene_ids))): # continue # end filter if gene_num == '-' or discontinued_num == '-': continue if self.testMode and int(gene_num) not in self.gene_ids: continue if not self.testMode and int(tax_num) not in self.tax_ids: continue line_counter += 1 gene_id = ':'.join(('NCBIGene', gene_num)) discontinued_gene_id = ':'.join(('NCBIGene', discontinued_num)) # add the two genes if self.class_or_indiv.get(gene_id) == 'C': gu.addClassToGraph(g, gene_id, None) gu.addClassToGraph( g, discontinued_gene_id, discontinued_symbol) # add the new gene id to replace the old gene id gu.addDeprecatedClass(g, discontinued_gene_id, [gene_id]) else: gu.addIndividualToGraph(g, gene_id, None) gu.addIndividualToGraph( g, discontinued_gene_id, discontinued_symbol) gu.addDeprecatedIndividual( g, discontinued_gene_id, [gene_id]) # also add the old symbol as a synonym of the new gene gu.addSynonym(g, gene_id, discontinued_symbol) if (not self.testMode) and\ (limit is not None and line_counter > limit): break return
def process_catalog(self, limit=None): """ :param limit: :return: """ raw = '/'.join((self.rawdir, self.files['catalog']['file'])) logger.info("Processing Data from %s", raw) gu = GraphUtils(curie_map.get()) if self.testMode: # set the graph to build g = self.testgraph else: g = self.graph line_counter = 0 geno = Genotype(g) gu.loadProperties(g, geno.object_properties, gu.OBJPROP) gu.loadAllProperties(g) tax_id = 'NCBITaxon:9606' # hardcode genome_version = 'GRCh38' # hardcode # build a hashmap of genomic location to identifiers, # to try to get the equivalences loc_to_id_hash = {} with open(raw, 'r', encoding="iso-8859-1") as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') next(filereader, None) # skip the header row for row in filereader: if not row: pass else: line_counter += 1 (date_added_to_catalog, pubmed_num, first_author, pub_date, journal, link, study_name, disease_or_trait, initial_sample_description, replicate_sample_description, region, chrom_num, chrom_pos, reported_gene_nums, mapped_gene, upstream_gene_num, downstream_gene_num, snp_gene_nums, upstream_gene_distance, downstream_gene_distance, strongest_snp_risk_allele, snps, merged, snp_id_current, context, intergenic_flag, risk_allele_frequency, pvalue, pvalue_mlog, pvalue_text, or_or_beta, confidence_interval_95, platform_with_snps_passing_qc, cnv_flag, mapped_trait, mapped_trait_uri) = row intersect = \ list(set([str(i) for i in self.test_ids['gene']]) & set(re.split(r',', snp_gene_nums))) # skip if no matches found in test set if self.testMode and len(intersect) == 0: continue # 06-May-2015 25917933 Zai CC 20-Nov-2014 J Psychiatr Res http://europepmc.org/abstract/MED/25917933 # A genome-wide association study of suicide severity scores in bipolar disorder. # Suicide in bipolar disorder # 959 European ancestry individuals NA # 10p11.22 10 32704340 C10orf68, CCDC7, ITGB1 CCDC7 # rs7079041-A rs7079041 0 7079041 intron 0 2E-6 5.698970 if chrom_num != '' and chrom_pos != '': loc = 'chr'+str(chrom_num)+':'+str(chrom_pos) if loc not in loc_to_id_hash: loc_to_id_hash[loc] = set() else: loc = None if re.search(r' x ', strongest_snp_risk_allele) \ or re.search(r',', strongest_snp_risk_allele): # TODO deal with haplotypes logger.warning( "We can't deal with haplotypes yet: %s", strongest_snp_risk_allele) continue elif re.match(r'rs', strongest_snp_risk_allele): rs_id = 'dbSNP:'+strongest_snp_risk_allele.strip() # remove the alteration elif re.match(r'kgp', strongest_snp_risk_allele): # FIXME this isn't correct rs_id = 'dbSNP:'+strongest_snp_risk_allele.strip() # http://www.1000genomes.org/faq/what-are-kgp-identifiers # for some information # They were created by Illumina for their genotyping # platform before some variants identified during the # pilot phase of the project had been assigned # rs numbers. elif re.match(r'chr', strongest_snp_risk_allele): # like: chr10:106180121-G rs_id = ':gwas-' + \ re.sub( r':', '-', strongest_snp_risk_allele.strip()) elif strongest_snp_risk_allele.strip() == '': # logger.debug( # "No strongest SNP risk allele for %s:\n%s", # pubmed_num, str(row)) # FIXME still consider adding in the EFO terms # for what the study measured? continue else: logger.warning( "There's a snp id i can't manage: %s", strongest_snp_risk_allele) continue alteration = re.search(r'-(.*)$', rs_id) if alteration is not None \ and re.match(r'[ATGC]', alteration.group(1)): # add variation to snp pass # TODO rs_id = re.sub(r'-.*$', '', rs_id).strip() if loc is not None: loc_to_id_hash[loc].add(rs_id) pubmed_id = 'PMID:'+pubmed_num r = Reference( pubmed_id, Reference.ref_types['journal_article']) r.addRefToGraph(g) # create the chromosome chrom_id = makeChromID(chrom_num, genome_version, 'CHR') # add the feature to the graph snp_description = None if risk_allele_frequency != '' and \ risk_allele_frequency != 'NR': snp_description = \ str(risk_allele_frequency) + \ ' [risk allele frequency]' f = Feature( rs_id, strongest_snp_risk_allele.strip(), Feature.types[r'SNP'], snp_description) if chrom_num != '' and chrom_pos != '': f.addFeatureStartLocation(chrom_pos, chrom_id) f.addFeatureEndLocation(chrom_pos, chrom_id) f.addFeatureToGraph(g) f.addTaxonToFeature(g, tax_id) # TODO consider adding allele frequency as property; # but would need background info to do that # also want to add other descriptive info about # the variant from the context for c in re.split(r';', context): cid = self._map_variant_type(c.strip()) if cid is not None: gu.addType(g, rs_id, cid) # add deprecation information if merged == 1 and str(snp_id_current.strip()) != '': # get the current rs_id current_rs_id = 'dbSNP:' if not re.match(r'rs', snp_id_current): current_rs_id += 'rs' if loc is not None: loc_to_id_hash[loc].append(current_rs_id) current_rs_id += str(snp_id_current) gu.addDeprecatedIndividual(g, rs_id, current_rs_id) # TODO check on this # should we add the annotations to the current # or orig? gu.makeLeader(g, current_rs_id) else: gu.makeLeader(g, rs_id) # add the feature as a sequence alteration # affecting various genes # note that intronic variations don't necessarily list # the genes such as for rs10448080 FIXME if snp_gene_nums != '': for s in re.split(r',', snp_gene_nums): s = s.strip() # still have to test for this, # because sometimes there's a leading comma if s != '': gene_id = 'NCBIGene:'+s geno.addAlleleOfGene(rs_id, gene_id) # add the up and downstream genes if they are available if upstream_gene_num != '': downstream_gene_id = 'NCBIGene:'+downstream_gene_num gu.addTriple( g, rs_id, Feature.object_properties[ r'upstream_of_sequence_of'], downstream_gene_id) if downstream_gene_num != '': upstream_gene_id = 'NCBIGene:'+upstream_gene_num gu.addTriple( g, rs_id, Feature.object_properties[ 'downstream_of_sequence_of'], upstream_gene_id) description = 'A study of ' + disease_or_trait + \ ' in ' + initial_sample_description if replicate_sample_description != '': description = \ ' '.join( (description, 'with', replicate_sample_description)) if platform_with_snps_passing_qc != '': description = ' '.join( (description, 'on platform', platform_with_snps_passing_qc)) description = ' '.join((description, '(p='+pvalue+')')) # make associations to the EFO terms; there can be >1 if mapped_trait_uri.strip() != '': for t in re.split(r',', mapped_trait_uri): t = t.strip() cu = CurieUtil(curie_map.get()) tid = cu.get_curie(t) assoc = G2PAssoc( self.name, rs_id, tid, gu.object_properties['contributes_to']) assoc.add_source(pubmed_id) # combinatorial evidence # used in automatic assertion eco_id = 'ECO:0000213' assoc.add_evidence(eco_id) # assoc.set_description(description) # FIXME score should get added to provenance/study # assoc.set_score(pvalue) assoc.add_association_to_graph(g) if not self.testMode and\ (limit is not None and line_counter > limit): break Assoc(self.name).load_all_properties(g) # loop through the location hash, # and make all snps at that location equivalent for l in loc_to_id_hash: snp_ids = loc_to_id_hash[l] if len(snp_ids) > 1: logger.info("%s has >1 snp id: %s", l, str(snp_ids)) return