def _add_snp_gene_relation(self, snp_id, snp_gene_nums, upstream_gene_num, downstream_gene_num): if self.testMode: g = self.testgraph else: g = self.graph geno = Genotype(g) # add the feature as a sequence alteration # affecting various genes # note that intronic variations don't necessarily list # the genes such as for rs10448080 FIXME if snp_gene_nums != '': for s in re.split(r',', snp_gene_nums): s = s.strip() # still have to test for this, # because sometimes there's a leading comma if s != '': gene_id = 'NCBIGene:' + s geno.addAffectedLocus(snp_id, gene_id) # add the up and downstream genes if they are available if upstream_gene_num != '': downstream_gene_id = 'NCBIGene:' + downstream_gene_num g.addTriple( snp_id, Feature.object_properties[ r'upstream_of_sequence_of'], downstream_gene_id) if downstream_gene_num != '': upstream_gene_id = 'NCBIGene:' + upstream_gene_num g.addTriple( snp_id, Feature.object_properties[ 'downstream_of_sequence_of'], upstream_gene_id)
def _add_snp_gene_relation(self, snp_id, snp_gene_nums, upstream_gene_num, downstream_gene_num): if self.test_mode: graph = self.testgraph else: graph = self.graph geno = Genotype(graph) # add the feature as a sequence alteration # affecting various genes # note that intronic variations don't necessarily list # the genes such as for rs10448080 FIXME if snp_gene_nums != '': for geneid in re.split(r',', snp_gene_nums): geneid = geneid.strip() # still have to test for this, # because sometimes there's a leading comma if geneid != '': geno.addAffectedLocus(snp_id, 'ENSEMBL:' + geneid) # add the up and downstream genes if they are available if downstream_gene_num != '': downstream_gene_id = 'ENSEMBL:' + downstream_gene_num graph.addTriple(snp_id, self.globaltt['is upstream of sequence of'], downstream_gene_id) if upstream_gene_num != '': upstream_gene_id = 'ENSEMBL:' + upstream_gene_num graph.addTriple(snp_id, self.globaltt['is downstream of sequence of'], upstream_gene_id)
def _process_phene_gene_row(self, row): geno = Genotype(self.graph) model = Model(self.graph) gene_id = self.id_hash['gene'].get(row['gene_id']) phene_id = self.id_hash['phene'].get(row['phene_id']) omia_id = self._get_omia_id_from_phene_id(phene_id) if self.testMode and not ( omia_id in self.test_ids['disease'] and row['gene_id'] in self.test_ids['gene']) or\ gene_id is None or phene_id is None: return # occasionally some phenes are missing! (ex: 406) if phene_id is None: LOG.warning("Phene id %s is missing", str(row['phene_id'])) return gene_label = self.label_hash[gene_id] # some variant of gene_id has phenotype d vl = '_:' + re.sub(r'NCBIGene:', '', str(gene_id)) + 'VL' geno.addAllele(vl, 'some variant of ' + gene_label) geno.addAlleleOfGene(vl, gene_id) geno.addAffectedLocus(vl, gene_id) model.addBlankNodeAnnotation(vl) assoc = G2PAssoc(self.graph, self.name, vl, phene_id) assoc.add_association_to_graph() # add the gene id to the set of annotated genes # for later lookup by orthology self.annotated_genes.add(gene_id) return
def _add_snp_gene_relation( self, snp_id, snp_gene_nums, upstream_gene_num, downstream_gene_num): if self.test_mode: graph = self.testgraph else: graph = self.graph geno = Genotype(graph) # add the feature as a sequence alteration # affecting various genes # note that intronic variations don't necessarily list # the genes such as for rs10448080 FIXME if snp_gene_nums != '': for geneid in re.split(r',', snp_gene_nums): geneid = geneid.strip() # still have to test for this, # because sometimes there's a leading comma if geneid != '': geno.addAffectedLocus(snp_id, 'NCBIGene:' + geneid) # add the up and downstream genes if they are available if upstream_gene_num != '': downstream_gene_id = 'NCBIGene:' + downstream_gene_num graph.addTriple( snp_id, self.globaltt['is upstream of sequence of'], downstream_gene_id) if downstream_gene_num != '': upstream_gene_id = 'NCBIGene:' + upstream_gene_num graph.addTriple( snp_id, self.globaltt['is downstream of sequence of'], upstream_gene_id)
def _add_snp_gene_relation(self, snp_id, snp_gene_nums, upstream_gene_num, downstream_gene_num): if self.testMode: g = self.testgraph else: g = self.graph geno = Genotype(g) # add the feature as a sequence alteration # affecting various genes # note that intronic variations don't necessarily list # the genes such as for rs10448080 FIXME if snp_gene_nums != '': for s in re.split(r',', snp_gene_nums): s = s.strip() # still have to test for this, # because sometimes there's a leading comma if s != '': gene_id = 'NCBIGene:' + s geno.addAffectedLocus(snp_id, gene_id) # add the up and downstream genes if they are available if upstream_gene_num != '': downstream_gene_id = 'NCBIGene:' + downstream_gene_num g.addTriple(snp_id, Feature.object_properties[r'upstream_of_sequence_of'], downstream_gene_id) if downstream_gene_num != '': upstream_gene_id = 'NCBIGene:' + upstream_gene_num g.addTriple(snp_id, Feature.object_properties['downstream_of_sequence_of'], upstream_gene_id)
def _build_gene_disease_model(self, gene_id, relation_id, disease_id, variant_label, consequence_predicate=None, consequence_id=None, allelic_requirement=None, pmids=None): """ Builds gene variant disease model :return: None """ model = Model(self.graph) geno = Genotype(self.graph) pmids = [] if pmids is None else pmids is_variant = False variant_or_gene = gene_id variant_id_string = variant_label variant_bnode = self.make_id(variant_id_string, "_") if consequence_predicate is not None \ and consequence_id is not None: is_variant = True model.addTriple(variant_bnode, consequence_predicate, consequence_id) # Hack to add labels to terms that # don't exist in an ontology if consequence_id.startswith(':'): model.addLabel(consequence_id, consequence_id.strip(':').replace('_', ' ')) if is_variant: variant_or_gene = variant_bnode # Typically we would type the variant using the # molecular consequence, but these are not specific # enough for us to make mappings (see translation table) model.addIndividualToGraph(variant_bnode, variant_label, self.globaltt['variant_locus']) geno.addAffectedLocus(variant_bnode, gene_id) model.addBlankNodeAnnotation(variant_bnode) assoc = G2PAssoc(self.graph, self.name, variant_or_gene, disease_id, relation_id) assoc.source = pmids assoc.add_association_to_graph() if allelic_requirement is not None and is_variant is False: model.addTriple(assoc.assoc_id, self.globaltt['has_allelic_requirement'], allelic_requirement) if allelic_requirement.startswith(':'): model.addLabel( allelic_requirement, allelic_requirement.strip(':').replace('_', ' '))
def process_disease_association(self, limit): raw = '/'.join((self.rawdir, self.files['disease_assoc']['file'])) if self.testMode: g = self.testgraph else: g = self.graph model = Model(g) logger.info("Processing disease models") geno = Genotype(g) line_counter = 0 worm_taxon = 'NCBITaxon:6239' with open(raw, 'r') as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in filereader: if re.match(r'!', ''.join(row)): # header continue line_counter += 1 (db, gene_num, gene_symbol, is_not, disease_id, ref, eco_symbol, with_or_from, aspect, gene_name, gene_synonym, gene_class, taxon, date, assigned_by, blank, blank2) = row if self.testMode and gene_num not in self.test_ids['gene']: continue # TODO add NOT phenotypes if is_not == 'NOT': continue # WB WBGene00000001 aap-1 DOID:2583 PMID:19029536 IEA ENSEMBL:ENSG00000145675|OMIM:615214 D Y110A7A.10 gene taxon:6239 20150612 WB gene_id = 'WormBase:'+gene_num # make a variant of the gene vl = '_:'+'-'.join((gene_num, 'unspecified')) vl_label = 'some variant of '+gene_symbol geno.addAffectedLocus(vl, gene_id) model.addBlankNodeAnnotation(vl) animal_id = geno.make_experimental_model_with_genotype( vl, vl_label, worm_taxon, 'worm') assoc = G2PAssoc( g, self.name, animal_id, disease_id, model.object_properties['model_of']) ref = re.sub(r'WB_REF:', 'WormBase:', ref) if ref != '': assoc.add_source(ref) eco_id = None if eco_symbol == 'IEA': eco_id = 'ECO:0000501' # IEA is this now if eco_id is not None: assoc.add_evidence(eco_id) assoc.add_association_to_graph() return
def process_disease_association(self, limit): raw = '/'.join((self.rawdir, self.files['disease_assoc']['file'])) if self.testMode: g = self.testgraph else: g = self.graph model = Model(g) logger.info("Processing disease models") geno = Genotype(g) line_counter = 0 worm_taxon = 'NCBITaxon:6239' with open(raw, 'r') as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in filereader: if re.match(r'!', ''.join(row)): # header continue line_counter += 1 (db, gene_num, gene_symbol, is_not, disease_id, ref, eco_symbol, with_or_from, aspect, gene_name, gene_synonym, gene_class, taxon, date, assigned_by, blank, blank2) = row if self.testMode and gene_num not in self.test_ids['gene']: continue # TODO add NOT phenotypes if is_not == 'NOT': continue # WB WBGene00000001 aap-1 DOID:2583 PMID:19029536 IEA ENSEMBL:ENSG00000145675|OMIM:615214 D Y110A7A.10 gene taxon:6239 20150612 WB gene_id = 'WormBase:' + gene_num # make a variant of the gene vl = '_:' + '-'.join((gene_num, 'unspecified')) vl_label = 'some variant of ' + gene_symbol geno.addAffectedLocus(vl, gene_id) model.addBlankNodeAnnotation(vl) animal_id = geno.make_experimental_model_with_genotype( vl, vl_label, worm_taxon, 'worm') assoc = G2PAssoc(g, self.name, animal_id, disease_id, model.object_properties['model_of']) ref = re.sub(r'WB_REF:', 'WormBase:', ref) if ref != '': assoc.add_source(ref) eco_id = None if eco_symbol == 'IEA': eco_id = 'ECO:0000501' # IEA is this now if eco_id is not None: assoc.add_evidence(eco_id) assoc.add_association_to_graph() return
def _make_pheno_assoc(self, g, gene_id, gene_symbol, disorder_num, disorder_label, phene_key): geno = Genotype(g) model = Model(g) disorder_id = ':'.join(('OMIM', disorder_num)) rel_id = model.object_properties['has_phenotype'] # default rel_label = 'causes' if re.match(r'\[', disorder_label): rel_id = model.object_properties['is_marker_for'] rel_label = 'is a marker for' elif re.match(r'\{', disorder_label): rel_id = model.object_properties['contributes_to'] rel_label = 'contributes to' elif re.match(r'\?', disorder_label): # this is a questionable mapping! skip? rel_id = model.object_properties['contributes_to'] rel_label = 'contributes to' evidence = self._map_phene_mapping_code_to_eco(phene_key) # we actually want the association between the gene and the disease # to be via an alternate locus not the "wildtype" gene itself. # so we make an anonymous alternate locus, # and put that in the association. # but we only need to do that in the cases when it's not an NCBIGene # (as that is a sequence feature itself) if re.match(r'OMIM:', gene_id): alt_locus = '_:' + re.sub(r':', '', gene_id) + '-' + disorder_num + 'VL' alt_label = gene_symbol.strip() if alt_label is not None and alt_label != '': alt_label = \ ' '.join(('some variant of', alt_label, 'that', rel_label, disorder_label)) else: alt_label = None model.addIndividualToGraph(alt_locus, alt_label, Genotype.genoparts['variant_locus']) geno.addAffectedLocus(alt_locus, gene_id) model.addBlankNodeAnnotation(alt_locus) else: # assume it's already been added alt_locus = gene_id assoc = G2PAssoc(g, self.name, alt_locus, disorder_id, rel_id) assoc.add_evidence(evidence) assoc.add_association_to_graph() return
def _make_pheno_assoc(self, g, gene_id, gene_symbol, disorder_num, disorder_label, phene_key): geno = Genotype(g) model = Model(g) disorder_id = ':'.join(('OMIM', disorder_num)) rel_id = model.object_properties['has_phenotype'] # default rel_label = 'causes' if re.match(r'\[', disorder_label): rel_id = model.object_properties['is_marker_for'] rel_label = 'is a marker for' elif re.match(r'\{', disorder_label): rel_id = model.object_properties['contributes_to'] rel_label = 'contributes to' elif re.match(r'\?', disorder_label): # this is a questionable mapping! skip? rel_id = model.object_properties['contributes_to'] rel_label = 'contributes to' evidence = self._map_phene_mapping_code_to_eco(phene_key) # we actually want the association between the gene and the disease # to be via an alternate locus not the "wildtype" gene itself. # so we make an anonymous alternate locus, # and put that in the association. # but we only need to do that in the cases when it's not an NCBIGene # (as that is a sequence feature itself) if re.match(r'OMIM:', gene_id): alt_locus = '_:'+re.sub(r':', '', gene_id)+'-'+disorder_num+'VL' alt_label = gene_symbol.strip() if alt_label is not None and alt_label != '': alt_label = \ ' '.join(('some variant of', alt_label, 'that', rel_label, disorder_label)) else: alt_label = None model.addIndividualToGraph( alt_locus, alt_label, Genotype.genoparts['variant_locus']) geno.addAffectedLocus(alt_locus, gene_id) model.addBlankNodeAnnotation(alt_locus) else: # assume it's already been added alt_locus = gene_id assoc = G2PAssoc(g, self.name, alt_locus, disorder_id, rel_id) assoc.add_evidence(evidence) assoc.add_association_to_graph() return
def _process_omim2gene(self, limit=None): """ This method maps the OMIM IDs and KEGG gene ID. Currently split based on the link_type field. Equivalent link types are mapped as gene XRefs. Reverse link types are mapped as disease to gene associations. Original link types are currently skipped. Triples created: <kegg_gene_id> is a Gene <omim_gene_id> is a Gene <kegg_gene_id>> hasXref <omim_gene_id> <assoc_id> has subject <omim_disease_id> <assoc_id> has object <kegg_gene_id> :param limit: :return: """ LOG.info("Processing OMIM to KEGG gene") if self.test_mode: graph = self.testgraph else: graph = self.graph model = Model(graph) line_counter = 0 geno = Genotype(graph) raw = '/'.join((self.rawdir, self.files['omim2gene']['file'])) with open(raw, 'r', encoding="iso-8859-1") as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in filereader: line_counter += 1 (kegg_gene_id, omim_id, link_type) = row if self.test_mode and kegg_gene_id not in self.test_ids['genes']: continue kegg_gene_id = 'KEGG-' + kegg_gene_id.strip() omim_id = re.sub(r'omim', 'OMIM', omim_id) if link_type == 'equivalent': # these are genes! # so add them as a class then make equivalence model.addClassToGraph(omim_id, None) geno.addGene(kegg_gene_id, None) if not DipperUtil.is_omim_disease(omim_id): model.addEquivalentClass(kegg_gene_id, omim_id) elif link_type == 'reverse': # make an association between an OMIM ID & the KEGG gene ID # we do this with omim ids because # they are more atomic than KEGG ids alt_locus_id = self._make_variant_locus_id(kegg_gene_id, omim_id) alt_label = self.label_hash[alt_locus_id] model.addIndividualToGraph( alt_locus_id, alt_label, self.globaltt['variant_locus']) geno.addAffectedLocus(alt_locus_id, kegg_gene_id) model.addBlankNodeAnnotation(alt_locus_id) # Add the disease to gene relationship. rel = self.globaltt['is marker for'] assoc = G2PAssoc(graph, self.name, alt_locus_id, omim_id, rel) assoc.add_association_to_graph() elif link_type == 'original': # these are sometimes a gene, and sometimes a disease LOG.info( 'Unable to handle original link for %s-%s', kegg_gene_id, omim_id) else: # don't know what these are LOG.warning( 'Unhandled link type for %s-%s: %s', kegg_gene_id, omim_id, link_type) if (not self.test_mode) and ( limit is not None and line_counter > limit): break LOG.info("Done with OMIM to KEGG gene") return
def _process_kegg_disease2gene(self, limit=None): """ This method creates an association between diseases and their associated genes. We are being conservative here, and only processing those diseases for which there is no mapping to OMIM. Triples created: <alternate_locus> is an Individual <alternate_locus> has type <variant_locus> <alternate_locus> is an allele of <gene_id> <assoc_id> has subject <disease_id> <assoc_id> has object <gene_id> :param limit: :return: """ LOG.info("Processing KEGG disease to gene") if self.test_mode: graph = self.testgraph else: graph = self.graph model = Model(graph) line_counter = 0 geno = Genotype(graph) rel = self.globaltt['is marker for'] noomimset = set() raw = '/'.join((self.rawdir, self.files['disease_gene']['file'])) with open(raw, 'r', encoding="iso-8859-1") as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in filereader: line_counter += 1 (gene_id, disease_id) = row if self.test_mode and gene_id not in self.test_ids['genes']: continue gene_id = 'KEGG-' + gene_id.strip() disease_id = 'KEGG-' + disease_id.strip() # only add diseases for which # there is no omim id and not a grouping class if disease_id not in self.kegg_disease_hash: # add as a class disease_label = None if disease_id in self.label_hash: disease_label = self.label_hash[disease_id] if re.search(r'includ', str(disease_label)): # they use 'including' when it's a grouping class LOG.info( "Skipping this association because " + "it's a grouping class: %s", disease_label) continue # type this disease_id as a disease model.addClassToGraph( disease_id, disease_label, self.globaltt['disease']) noomimset.add(disease_id) alt_locus_id = self._make_variant_locus_id(gene_id, disease_id) alt_label = self.label_hash[alt_locus_id] model.addIndividualToGraph( alt_locus_id, alt_label, self.globaltt['variant_locus']) geno.addAffectedLocus(alt_locus_id, gene_id) model.addBlankNodeAnnotation(alt_locus_id) # Add the disease to gene relationship. assoc = G2PAssoc(graph, self.name, alt_locus_id, disease_id, rel) assoc.add_association_to_graph() if (not self.test_mode) and (limit is not None and line_counter > limit): break LOG.info("Done with KEGG disease to gene") LOG.info("Found %d diseases with no omim id", len(noomimset)) return
def _process_qtls_genetic_location( self, raw, txid, common_name, limit=None): """ This function processes Triples created: :param limit: :return: """ aql_curie = self.files[common_name + '_cm']['curie'] if self.test_mode: graph = self.testgraph else: graph = self.graph line_counter = 0 geno = Genotype(graph) model = Model(graph) eco_id = self.globaltt['quantitative trait analysis evidence'] taxon_curie = 'NCBITaxon:' + txid LOG.info("Processing genetic location for %s from %s", taxon_curie, raw) with open(raw, 'r', encoding="iso-8859-1") as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in filereader: line_counter += 1 (qtl_id, qtl_symbol, trait_name, assotype, empty, chromosome, position_cm, range_cm, flankmark_a2, flankmark_a1, peak_mark, flankmark_b1, flankmark_b2, exp_id, model_id, test_base, sig_level, lod_score, ls_mean, p_values, f_statistics, variance, bayes_value, likelihood_ratio, trait_id, dom_effect, add_effect, pubmed_id, gene_id, gene_id_src, gene_id_type, empty2) = row if self.test_mode and int(qtl_id) not in self.test_ids: continue qtl_id = common_name + 'QTL:' + qtl_id.strip() trait_id = ':'.join((aql_curie, trait_id.strip())) # Add QTL to graph feature = Feature(graph, qtl_id, qtl_symbol, self.globaltt['QTL']) feature.addTaxonToFeature(taxon_curie) # deal with the chromosome chrom_id = makeChromID(chromosome, taxon_curie, 'CHR') # add a version of the chromosome which is defined as # the genetic map build_id = 'MONARCH:'+common_name.strip()+'-linkage' build_label = common_name+' genetic map' geno.addReferenceGenome(build_id, build_label, taxon_curie) chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH') geno.addChromosomeInstance( chromosome, build_id, build_label, chrom_id) start = stop = None # range_cm sometimes ends in "(Mb)" (i.e pig 2016 Nov) range_mb = re.split(r'\(', range_cm) if range_mb is not None: range_cm = range_mb[0] if re.search(r'[0-9].*-.*[0-9]', range_cm): range_parts = re.split(r'-', range_cm) # check for poorly formed ranges if len(range_parts) == 2 and\ range_parts[0] != '' and range_parts[1] != '': (start, stop) = [ int(float(x.strip())) for x in re.split(r'-', range_cm)] else: LOG.info( "A cM range we can't handle for QTL %s: %s", qtl_id, range_cm) elif position_cm != '': match = re.match(r'([0-9]*\.[0-9]*)', position_cm) if match is not None: position_cm = match.group() start = stop = int(float(position_cm)) # FIXME remove converion to int for start/stop # when schema can handle floats add in the genetic location # based on the range feature.addFeatureStartLocation( start, chrom_in_build_id, None, [self.globaltt['FuzzyPosition']]) feature.addFeatureEndLocation( stop, chrom_in_build_id, None, [self.globaltt['FuzzyPosition']]) feature.addFeatureToGraph() # sometimes there's a peak marker, like a rsid. # we want to add that as a variant of the gene, # and xref it to the qtl. dbsnp_id = None if peak_mark != '' and peak_mark != '.' and \ re.match(r'rs', peak_mark.strip()): dbsnp_id = 'dbSNP:'+peak_mark.strip() model.addIndividualToGraph( dbsnp_id, None, self.globaltt['sequence_alteration']) model.addXref(qtl_id, dbsnp_id) gene_id = gene_id.replace('uncharacterized ', '').strip() if gene_id is not None and gene_id != '' and gene_id != '.'\ and re.fullmatch(r'[^ ]*', gene_id) is not None: # we assume if no src is provided and gene_id is an integer, # then it is an NCBI gene ... (okay, lets crank that back a notch) if gene_id_src == '' and gene_id.isdigit() and \ gene_id in self.gene_info: # LOG.info( # 'Warm & Fuzzy saying %s is a NCBI gene for %s', # gene_id, common_name) gene_id_src = 'NCBIgene' elif gene_id_src == '' and gene_id.isdigit(): LOG.warning( 'Cold & Prickely saying %s is a NCBI gene for %s', gene_id, common_name) gene_id_src = 'NCBIgene' elif gene_id_src == '': LOG.error( ' "%s" is a NOT NCBI gene for %s', gene_id, common_name) gene_id_src = None if gene_id_src == 'NCBIgene': gene_id = 'NCBIGene:' + gene_id # we will expect that these will get labels elsewhere geno.addGene(gene_id, None) # FIXME what is the right relationship here? geno.addAffectedLocus(qtl_id, gene_id) if dbsnp_id is not None: # add the rsid as a seq alt of the gene_id vl_id = '_:' + re.sub( r':', '', gene_id) + '-' + peak_mark.strip() geno.addSequenceAlterationToVariantLocus( dbsnp_id, vl_id) geno.addAffectedLocus(vl_id, gene_id) # add the trait model.addClassToGraph(trait_id, trait_name) # Add publication reference = None if re.match(r'ISU.*', pubmed_id): pub_id = 'AQTLPub:'+pubmed_id.strip() reference = Reference(graph, pub_id) elif pubmed_id != '': pub_id = 'PMID:' + pubmed_id.strip() reference = Reference( graph, pub_id, self.globaltt['journal article']) if reference is not None: reference.addRefToGraph() # make the association to the QTL assoc = G2PAssoc( graph, self.name, qtl_id, trait_id, self.globaltt['is marker for']) assoc.add_evidence(eco_id) assoc.add_source(pub_id) # create a description from the contents of the file # desc = '' # assoc.addDescription(g, assoc_id, desc) # TODO add exp_id as evidence # if exp_id != '': # exp_id = 'AQTLExp:'+exp_id # gu.addIndividualToGraph(g, exp_id, None, eco_id) if p_values != '': scr = re.sub(r'<', '', p_values) scr = re.sub(r',', '.', scr) # international notation if scr.isnumeric(): score = float(scr) assoc.set_score(score) # todo add score type # TODO add LOD score? assoc.add_association_to_graph() # make the association to the dbsnp_id, if found if dbsnp_id is not None: # make the association to the dbsnp_id assoc = G2PAssoc( graph, self.name, dbsnp_id, trait_id, self.globaltt['is marker for']) assoc.add_evidence(eco_id) assoc.add_source(pub_id) # create a description from the contents of the file # desc = '' # assoc.addDescription(g, assoc_id, desc) # TODO add exp_id # if exp_id != '': # exp_id = 'AQTLExp:'+exp_id # gu.addIndividualToGraph(g, exp_id, None, eco_id) if p_values != '': scr = re.sub(r'<', '', p_values) scr = re.sub(r',', '.', scr) if scr.isnumeric(): score = float(scr) assoc.set_score(score) # todo add score type # TODO add LOD score? assoc.add_association_to_graph() if not self.test_mode and limit is not None and line_counter > limit: break LOG.info("Done with QTL genetic info") return
def _process_qtls_genetic_location( self, raw, src_key, txid, common_name, limit=None): """ This function processes Triples created: :param limit: :return: """ aql_curie = self.files[src_key]['curie'] common_name = common_name.strip() if self.test_mode: graph = self.testgraph else: graph = self.graph geno = Genotype(graph) model = Model(graph) eco_id = self.globaltt['quantitative trait analysis evidence'] taxon_curie = 'NCBITaxon:' + txid LOG.info("Processing genetic location for %s from %s", taxon_curie, raw) with open(raw, 'r', encoding="iso-8859-1") as csvfile: reader = csv.reader(csvfile, delimiter='\t', quotechar='\"') # no header in these files, so no header checking col = self.files[src_key]['columns'] col_len = len(col) for row in reader: if len(row) != col_len and ''.join(row[col_len:]) != '': LOG.warning( "Problem parsing %s line %i containing: \n%s\n" "got %i cols but expected %i", raw, reader.line_num, row, len(row), col_len) # LOG.info(row) continue qtl_id = row[col.index('QTL_ID')].strip() qtl_symbol = row[col.index('QTL_symbol')].strip() trait_name = row[col.index('Trait_name')].strip() # assotype = row[col.index('assotype')].strip() chromosome = row[col.index('Chromosome')].strip() position_cm = row[col.index('Position_cm')].strip() range_cm = row[col.index('range_cm')].strip() # flankmark_a2 = row[col.index('FlankMark_A2')].strip() # flankmark_a1 = row[col.index('FlankMark_A1')].strip() peak_mark = row[col.index('Peak_Mark')].strip() # flankmark_b1 = row[col.index('FlankMark_B1')].strip() # flankmark_b2 = row[col.index('FlankMark_B2')].strip() # exp_id = row[col.index('Exp_ID')].strip() # model_id = row[col.index('Model')].strip() # test_base = row[col.index('testbase')].strip() # sig_level = row[col.index('siglevel')].strip() # lod_score = row[col.index('LOD_score')].strip() # ls_mean = row[col.index('LS_mean')].strip() p_values = row[col.index('P_values')].strip() # f_statistics = row[col.index('F_Statistics')].strip() # variance = row[col.index('VARIANCE')].strip() # bayes_value = row[col.index('Bayes_value')].strip() # likelihood_ratio = row[col.index('LikelihoodR')].strip() trait_id = row[col.index('TRAIT_ID')].strip() # dom_effect = row[col.index('Dom_effect')].strip() # add_effect = row[col.index('Add_effect')].strip() pubmed_id = row[col.index('PUBMED_ID')].strip() gene_id = row[col.index('geneID')].strip() gene_id_src = row[col.index('geneIDsrc')].strip() # gene_id_type = row[col.index('geneIDtype')].strip() if self.test_mode and int(qtl_id) not in self.test_ids: continue qtl_id = common_name + 'QTL:' + qtl_id.strip() trait_id = ':'.join((aql_curie, trait_id.strip())) # Add QTL to graph feature = Feature(graph, qtl_id, qtl_symbol, self.globaltt['QTL']) feature.addTaxonToFeature(taxon_curie) # deal with the chromosome chrom_id = makeChromID(chromosome, taxon_curie, 'CHR') # add a version of the chromosome which is defined as # the genetic map build_id = 'MONARCH:' + common_name + '-linkage' build_label = common_name + ' genetic map' geno.addReferenceGenome(build_id, build_label, taxon_curie) chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH') geno.addChromosomeInstance( chromosome, build_id, build_label, chrom_id) start = stop = None # range_cm sometimes ends in "(Mb)" (i.e pig 2016 Nov) range_mb = re.split(r'\(', range_cm) if range_mb is not None: range_cm = range_mb[0] if re.search(r'[0-9].*-.*[0-9]', range_cm): range_parts = re.split(r'-', range_cm) # check for poorly formed ranges if len(range_parts) == 2 and\ range_parts[0] != '' and range_parts[1] != '': (start, stop) = [ int(float(x.strip())) for x in re.split(r'-', range_cm)] else: LOG.info( "A cM range we can't handle for QTL %s: %s", qtl_id, range_cm) elif position_cm != '': match = re.match(r'([0-9]*\.[0-9]*)', position_cm) if match is not None: position_cm = match.group() start = stop = int(float(position_cm)) # FIXME remove converion to int for start/stop # when schema can handle floats add in the genetic location # based on the range feature.addFeatureStartLocation( start, chrom_in_build_id, None, [self.globaltt['FuzzyPosition']]) feature.addFeatureEndLocation( stop, chrom_in_build_id, None, [self.globaltt['FuzzyPosition']]) feature.addFeatureToGraph() # sometimes there's a peak marker, like a rsid. # we want to add that as a variant of the gene, # and xref it to the qtl. dbsnp_id = None if peak_mark != '' and peak_mark != '.' and \ re.match(r'rs', peak_mark.strip()): dbsnp_id = 'dbSNP:' + peak_mark.strip() model.addIndividualToGraph( dbsnp_id, None, self.globaltt['sequence_alteration']) model.addXref( qtl_id, dbsnp_id, xref_category=blv.terms['SequenceVariant']) gene_id = gene_id.replace('uncharacterized ', '').strip() gene_id = gene_id.strip(',') # for "100157483," in pig_QTLdata.txt if gene_id is not None and gene_id != '' and gene_id != '.'\ and re.fullmatch(r'[^ ]*', gene_id) is not None: # we assume if no src is provided and gene_id is an integer, # then it is an NCBI gene ... (okay, lets crank that back a notch) if gene_id_src == '' and gene_id.isdigit() and \ gene_id in self.gene_info: # LOG.info( # 'Warm & Fuzzy saying %s is a NCBI gene for %s', # gene_id, common_name) gene_id_src = 'NCBIgene' elif gene_id_src == '' and gene_id.isdigit(): LOG.warning( 'Cold & Prickely saying %s is a NCBI gene for %s', gene_id, common_name) gene_id_src = 'NCBIgene' elif gene_id_src == '': LOG.error( ' "%s" is a NOT NCBI gene for %s', gene_id, common_name) gene_id_src = None if gene_id_src == 'NCBIgene': gene_id = 'NCBIGene:' + gene_id # we will expect that these will get labels elsewhere geno.addGene(gene_id, None) # FIXME what is the right relationship here? geno.addAffectedLocus(qtl_id, gene_id) if dbsnp_id is not None: # add the rsid as a seq alt of the gene_id as a bnode vl_id = self.make_id(re.sub( r':', '', gene_id) + '-' + peak_mark.strip(), '_') geno.addSequenceAlterationToVariantLocus(dbsnp_id, vl_id) geno.addAffectedLocus(vl_id, gene_id) # add the trait model.addClassToGraph( trait_id, trait_name, class_category=blv.terms['PhenotypicFeature']) # Add publication reference = None if re.match(r'ISU.*', pubmed_id): pub_id = 'AQTLPub:' + pubmed_id.strip() reference = Reference(graph, pub_id) elif pubmed_id != '': pub_id = 'PMID:' + pubmed_id.strip() reference = Reference( graph, pub_id, self.globaltt['journal article']) if reference is not None: reference.addRefToGraph() # make the association to the QTL assoc = G2PAssoc( graph, self.name, qtl_id, trait_id, self.globaltt['is marker for']) assoc.add_evidence(eco_id) assoc.add_source(pub_id) # create a description from the contents of the file # desc = '' # assoc.addDescription(g, assoc_id, desc) # TODO add exp_id as evidence # if exp_id != '': # exp_id = 'AQTLExp:'+exp_id # gu.addIndividualToGraph(g, exp_id, None, eco_id) if p_values != '': scr = re.sub(r'<', '', p_values) scr = re.sub(r',', '.', scr) # international notation if scr.isnumeric(): score = float(scr) assoc.set_score(score) # todo add score type # TODO add LOD score? assoc.add_association_to_graph() # make the association to the dbsnp_id, if found if dbsnp_id is not None: # make the association to the dbsnp_id assoc = G2PAssoc( graph, self.name, dbsnp_id, trait_id, self.globaltt['is marker for']) assoc.add_evidence(eco_id) assoc.add_source(pub_id) # create a description from the contents of the file # desc = '' # assoc.addDescription(g, assoc_id, desc) # TODO add exp_id # if exp_id != '': # exp_id = 'AQTLExp:'+exp_id # gu.addIndividualToGraph(g, exp_id, None, eco_id) if p_values != '': scr = re.sub(r'<', '', p_values) scr = re.sub(r',', '.', scr) if scr.isnumeric(): score = float(scr) assoc.set_score(score) # todo add score type # TODO add LOD score? assoc.add_association_to_graph() # off by one - the following actually gives us (limit + 1) records if not self.test_mode and limit is not None and reader.line_num > limit: break LOG.info("Done with QTL genetic info")
def _process_haplotype(self, hap_id, hap_label, chrom_num, chrom_pos, context, risk_allele_frequency, mapped_gene, so_ontology): if self.test_mode: graph = self.testgraph else: graph = self.graph geno = Genotype(graph) model = Model(graph) # add the feature to the graph hap_description = None if risk_allele_frequency != '' and risk_allele_frequency != 'NR': hap_description = str( risk_allele_frequency) + ' [risk allele frequency]' model.addIndividualToGraph(hap_id, hap_label.strip(), self.globaltt['haplotype'], hap_description) geno.addTaxon(self.globaltt["H**o sapiens"], hap_id) snp_labels = re.split(r';\s?', hap_label) chrom_nums = re.split(r';\s?', chrom_num) chrom_positions = re.split(r';\s?', chrom_pos) context_list = re.split(r';\s?', context) mapped_genes = re.split(r';\s?', mapped_gene) snp_curies = list() for index, snp in enumerate(snp_labels): snp_curie, snp_type = self._get_curie_and_type_from_id(snp) if snp_type is None: # make blank node snp_curie = self.make_id(snp, "_") graph.addTriple(hap_id, self.globaltt['has_variant_part'], snp_curie) snp_curies.append(snp_curie) # courtesy http://stackoverflow.com/a/16720915 length = len(snp_labels) if not all( len(lst) == length for lst in [chrom_nums, chrom_positions, context_list]): LOG.warning( "Unexpected data field for haplotype %s \n " "will not add snp details", hap_label) return variant_in_gene_count = 0 for index, snp_curie in enumerate(snp_curies): self._add_snp_to_graph(snp_curie, snp_labels[index], chrom_nums[index], chrom_positions[index], context_list[index]) if len(mapped_genes) == len(snp_labels): so_class = self.resolve(context_list[index]) # removed the '+' for recursive one-or-more rdfs:subClassOf paths # just so it did not return an empty graph so_query = """ SELECT ?variant_label WHERE {{ {0} rdfs:subClassOf {1} ; rdfs:label ?variant_label . }} """.format(so_class, self.globaltt['gene_variant']) query_result = so_ontology.query(so_query) if len(list(query_result)) == 1: gene_id = DipperUtil.get_ncbi_id_from_symbol( mapped_genes[index]) if gene_id is not None: geno.addAffectedLocus(snp_curie, gene_id) geno.addAffectedLocus(hap_id, gene_id) variant_in_gene_count += 1 gene_id = DipperUtil.get_ncbi_id_from_symbol( mapped_genes[index]) if gene_id is not None: graph.addTriple(snp_curie, self.resolve(context_list[index]), gene_id) else: LOG.warning( "More mapped genes than snps, cannot disambiguate for %s", hap_label) # Seperate in case we want to apply a different relation # If not this is redundant with triples added above if len(mapped_genes) == variant_in_gene_count and len( set(mapped_genes)) == 1: gene_id = DipperUtil.get_ncbi_id_from_symbol(mapped_genes[0]) geno.addAffectedLocus(hap_id, gene_id) return
def _process_kegg_disease2gene(self, limit=None): """ This method creates an association between diseases and their associated genes. We are being conservative here, and only processing those diseases for which there is no mapping to OMIM. Triples created: <alternate_locus> is an Individual <alternate_locus> has type <variant_locus> <alternate_locus> is an allele of <gene_id> <assoc_id> has subject <disease_id> <assoc_id> has object <gene_id> :param limit: :return: """ LOG.info("Processing KEGG disease to gene") if self.test_mode: graph = self.testgraph else: graph = self.graph model = Model(graph) geno = Genotype(graph) rel = self.globaltt['is marker for'] noomimset = set() raw = '/'.join((self.rawdir, self.files['disease_gene']['file'])) with open(raw, 'r', encoding="iso-8859-1") as csvfile: reader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in reader: (gene_id, disease_id) = row if self.test_mode and gene_id not in self.test_ids['genes']: continue gene_id = 'KEGG-' + gene_id.strip() disease_id = 'KEGG-' + disease_id.strip() # only add diseases for which # there is no omim id and not a grouping class if disease_id not in self.kegg_disease_hash: # add as a class disease_label = None if disease_id in self.label_hash: disease_label = self.label_hash[disease_id] if re.search(r'includ', str(disease_label)): # they use 'including' when it's a grouping class LOG.info( "Skipping association because it's a grouping class: %s", disease_label) continue # type this disease_id as a disease model.addClassToGraph(disease_id, disease_label) # , class_type=self.globaltt['disease']) noomimset.add(disease_id) alt_locus_id = self._make_variant_locus_id(gene_id, disease_id) alt_label = self.label_hash[alt_locus_id] model.addIndividualToGraph( alt_locus_id, alt_label, self.globaltt['variant_locus']) geno.addAffectedLocus(alt_locus_id, gene_id) model.addBlankNodeAnnotation(alt_locus_id) # Add the disease to gene relationship. assoc = G2PAssoc(graph, self.name, alt_locus_id, disease_id, rel) assoc.add_association_to_graph() if not self.test_mode and ( limit is not None and reader.line_num > limit): break LOG.info("Done with KEGG disease to gene") LOG.info("Found %d diseases with no omim id", len(noomimset))
def _process_diseasegene(self, limit): """ :param limit: :return: """ if self.testMode: g = self.testgraph else: g = self.graph line_counter = 0 geno = Genotype(g) model = Model(g) myfile = '/'.join((self.rawdir, self.files['disease-gene']['file'])) # PYLINT complains iterparse deprecated, # but as of py 3.4 only the optional & unsupplied parse arg is. for event, elem in ET.iterparse(myfile): if elem.tag == 'Disorder': # get the element name and id, ignoreS element name # id = elem.get('id') # some internal identifier disorder_num = elem.find('OrphaNumber').text disorder_id = 'Orphanet:'+str(disorder_num) if self.testMode and \ disorder_id not in \ config.get_config()['test_ids']['disease']: continue disorder_label = elem.find('Name').text # make a hash of internal gene id to type for later lookup gene_iid_to_type = {} gene_list = elem.find('GeneList') for gene in gene_list.findall('Gene'): gene_iid = gene.get('id') gene_type = gene.find('GeneType').get('id') gene_iid_to_type[gene_iid] = gene_type # assuming that these are in the ontology model.addClassToGraph(disorder_id, disorder_label) assoc_list = elem.find('DisorderGeneAssociationList') for a in assoc_list.findall('DisorderGeneAssociation'): gene_iid = a.find('.//Gene').get('id') gene_name = a.find('.//Gene/Name').text gene_symbol = a.find('.//Gene/Symbol').text gene_num = a.find('./Gene/OrphaNumber').text gene_id = 'Orphanet:'+str(gene_num) gene_type_id = \ self._map_gene_type_id(gene_iid_to_type[gene_iid]) model.addClassToGraph( gene_id, gene_symbol, gene_type_id, gene_name) syn_list = a.find('./Gene/SynonymList') if int(syn_list.get('count')) > 0: for s in syn_list.findall('./Synonym'): model.addSynonym(gene_id, s.text) dgtype = a.find('DisorderGeneAssociationType').get('id') rel_id = self._map_rel_id(dgtype) dg_label = \ a.find('./DisorderGeneAssociationType/Name').text if rel_id is None: logger.warning( "Cannot map association type (%s) to RO " + "for association (%s | %s). Skipping.", dg_label, disorder_label, gene_symbol) continue alt_locus_id = '_:'+gene_num+'-'+disorder_num+'VL' alt_label = \ ' '.join(('some variant of', gene_symbol.strip(), 'that is a', dg_label.lower(), disorder_label)) model.addIndividualToGraph(alt_locus_id, alt_label, geno.genoparts['variant_locus']) geno.addAffectedLocus(alt_locus_id, gene_id) model.addBlankNodeAnnotation(alt_locus_id) # consider typing the gain/loss-of-function variants like: # http://sequenceontology.org/browser/current_svn/term/SO:0002054 # http://sequenceontology.org/browser/current_svn/term/SO:0002053 # use "assessed" status to issue an evidence code # FIXME I think that these codes are sub-optimal status_code = \ a.find('DisorderGeneAssociationStatus').get('id') # imported automatically asserted information # used in automatic assertion eco_id = 'ECO:0000323' # Assessed # TODO are these internal ids stable between releases? if status_code == '17991': # imported manually asserted information # used in automatic assertion eco_id = 'ECO:0000322' # Non-traceable author statement ECO_0000034 # imported information in automatic assertion ECO_0000313 assoc = G2PAssoc(g, self.name, alt_locus_id, disorder_id, rel_id) assoc.add_evidence(eco_id) assoc.add_association_to_graph() rlist = a.find('./Gene/ExternalReferenceList') eqid = None for r in rlist.findall('ExternalReference'): if r.find('Source').text == 'Ensembl': eqid = 'ENSEMBL:'+r.find('Reference').text elif r.find('Source').text == 'HGNC': eqid = 'HGNC:'+r.find('Reference').text elif r.find('Source').text == 'OMIM': eqid = 'OMIM:'+r.find('Reference').text else: pass # skip the others for now if eqid is not None: model.addClassToGraph(eqid, None) model.addEquivalentClass(gene_id, eqid) elem.clear() # empty the element if self.testMode and limit is not None and line_counter > limit: return return
class CTD(Source): """ The Comparative Toxicogenomics Database (CTD) includes curated data describing cross-species chemical–gene/protein interactions and chemical– and gene–disease associations to illuminate molecular mechanisms underlying variable susceptibility and environmentally influenced diseases. Here, we fetch, parse, and convert data from CTD into triples, leveraging only the associations based on DIRECT evidence (not using the inferred associations). We currently process the following associations: * chemical-disease * gene-pathway * gene-disease CTD curates relationships between genes and chemicals/diseases with marker/mechanism and/or therapeutic. Unfortunately, we cannot disambiguate between marker (gene expression) and mechanism (causation) for these associations. Therefore, we are left to relate these simply by "marker". CTD also pulls in genes and pathway membership from KEGG and REACTOME. We create groups of these following the pattern that the specific pathway is a subclass of 'cellular process' (a go process), and the gene is "involved in" that process. For diseases, we preferentially use OMIM identifiers when they can be used uniquely over MESH. Otherwise, we use MESH ids. Note that we scrub the following identifiers and their associated data: * REACT:REACT_116125 - generic disease class * MESH:D004283 - dog diseases * MESH:D004195 - disease models, animal * MESH:D030342 - genetic diseases, inborn * MESH:D040181 - genetic dieases, x-linked * MESH:D020022 - genetic predisposition to a disease """ files = { 'chemical_disease_interactions': { 'file': 'CTD_chemicals_diseases.tsv.gz', 'url': 'http://ctdbase.org/reports/CTD_chemicals_diseases.tsv.gz' }, 'gene_pathway': { 'file': 'CTD_genes_pathways.tsv.gz', 'url': 'http://ctdbase.org/reports/CTD_genes_pathways.tsv.gz' }, 'gene_disease': { 'file': 'CTD_genes_diseases.tsv.gz', 'url': 'http://ctdbase.org/reports/CTD_genes_diseases.tsv.gz' } } static_files = { 'publications': {'file': 'CTD_curated_references.tsv'} } def __init__(self, graph_type, are_bnodes_skolemized): super().__init__(graph_type, are_bnodes_skolemized, 'ctd') self.dataset = Dataset( 'ctd', 'CTD', 'http://ctdbase.org', None, 'http://ctdbase.org/about/legal.jsp') if 'test_ids' not in config.get_config() \ or 'gene' not in config.get_config()['test_ids']: logger.warning("not configured with gene test ids.") self.test_geneids = [] else: self.test_geneids = config.get_config()['test_ids']['gene'] if 'test_ids' not in config.get_config() \ or 'disease' not in config.get_config()['test_ids']: logger.warning("not configured with disease test ids.") self.test_diseaseids = [] else: self.test_diseaseids = config.get_config()['test_ids']['disease'] self.g = self.graph self.geno = Genotype(self.graph) self.pathway = Pathway(self.graph) return def fetch(self, is_dl_forced=False): """ Override Source.fetch() Fetches resources from CTD using the CTD.files dictionary Args: :param is_dl_forced (bool): Force download Returns: :return None """ self.get_files(is_dl_forced) self._fetch_disambiguating_assoc() # consider creating subsets of the files that # only have direct annotations (not inferred) return def parse(self, limit=None): """ Override Source.parse() Parses version and interaction information from CTD Args: :param limit (int, optional) limit the number of rows processed Returns: :return None """ if limit is not None: logger.info("Only parsing first %d rows", limit) logger.info("Parsing files...") # pub_map = dict() # file_path = '/'.join((self.rawdir, # self.static_files['publications']['file'])) # if os.path.exists(file_path) is True: # pub_map = self._parse_publication_file( # self.static_files['publications']['file'] # ) if self.testOnly: self.testMode = True if self.testMode: self.g = self.testgraph else: self.g = self.graph self.geno = Genotype(self.g) self.pathway = Pathway(self.g) self._parse_ctd_file( limit, self.files['chemical_disease_interactions']['file']) self._parse_ctd_file(limit, self.files['gene_pathway']['file']) self._parse_ctd_file(limit, self.files['gene_disease']['file']) self._parse_curated_chem_disease(limit) logger.info("Done parsing files.") return def _parse_ctd_file(self, limit, file): """ Parses files in CTD.files dictionary Args: :param limit (int): limit the number of rows processed :param file (str): file name (must be defined in CTD.file) Returns: :return None """ row_count = 0 version_pattern = re.compile(r'^# Report created: (.+)$') is_versioned = False file_path = '/'.join((self.rawdir, file)) with gzip.open(file_path, 'rt') as tsvfile: reader = csv.reader(tsvfile, delimiter="\t") for row in reader: # Scan the header lines until we get the version # There is no official version sp we are using # the upload timestamp instead if is_versioned is False: match = re.match(version_pattern, ' '.join(row)) if match: version = re.sub(r'\s|:', '-', match.group(1)) # TODO convert this timestamp to a proper timestamp self.dataset.setVersion(version) is_versioned = True elif re.match(r'^#', ' '.join(row)): pass else: row_count += 1 if file == self.files[ 'chemical_disease_interactions']['file']: self._process_interactions(row) elif file == self.files['gene_pathway']['file']: self._process_pathway(row) elif file == self.files['gene_disease']['file']: self._process_disease2gene(row) if not self.testMode and \ limit is not None and row_count >= limit: break return def _process_pathway(self, row): """ Process row of CTD data from CTD_genes_pathways.tsv.gz and generate triples Args: :param row (list): row of CTD data Returns: :return None """ model = Model(self.g) self._check_list_len(row, 4) (gene_symbol, gene_id, pathway_name, pathway_id) = row if self.testMode and (int(gene_id) not in self.test_geneids): return entrez_id = 'NCBIGene:' + gene_id pathways_to_scrub = [ 'REACT:REACT_116125', # disease "REACT:REACT_111045", # developmental biology "REACT:REACT_200794", # Mus musculus biological processes "REACT:REACT_13685"] # neuronal system ? if pathway_id in pathways_to_scrub: # these are lame "pathways" like generic # "disease" and "developmental biology" return # convert KEGG pathway ids... KEGG:12345 --> KEGG-path:map12345 if re.match(r'KEGG', pathway_id): pathway_id = re.sub(r'KEGG:', 'KEGG-path:map', pathway_id) # just in case, add it as a class model.addClassToGraph(entrez_id, None) self.pathway.addPathway(pathway_id, pathway_name) self.pathway.addGeneToPathway(entrez_id, pathway_id) return def _fetch_disambiguating_assoc(self): """ For any of the items in the chemical-disease association file that have ambiguous association types we fetch the disambiguated associations using the batch query API, and store these in a file. Elsewhere, we can loop through the file and create the appropriate associations. :return: """ disambig_file = '/'.join( (self.rawdir, self.static_files['publications']['file'])) assoc_file = '/'.join( (self.rawdir, self.files['chemical_disease_interactions']['file'])) # check if there is a local association file, # and download if it's dated later than the original intxn file if os.path.exists(disambig_file): dfile_dt = os.stat(disambig_file) afile_dt = os.stat(assoc_file) if dfile_dt < afile_dt: logger.info( "Local file date before chem-disease assoc file. " " Downloading...") else: logger.info( "Local file date after chem-disease assoc file. " " Skipping download.") return all_pubs = set() dual_evidence = re.compile(r'^marker\/mechanism\|therapeutic$') # first get all the unique publications with gzip.open(assoc_file, 'rt') as tsvfile: reader = csv.reader(tsvfile, delimiter="\t") for row in reader: if re.match(r'^#', ' '.join(row)): continue self._check_list_len(row, 10) (chem_name, chem_id, cas_rn, disease_name, disease_id, direct_evidence, inferred_gene_symbol, inference_score, omim_ids, pubmed_ids) = row if direct_evidence == '' or not \ re.match(dual_evidence, direct_evidence): continue if pubmed_ids is not None and pubmed_ids != '': all_pubs.update(set(re.split(r'\|', pubmed_ids))) sorted_pubs = sorted(list(all_pubs)) # now in batches of 4000, we fetch the chemical-disease associations batch_size = 4000 params = { 'inputType': 'reference', 'report': 'diseases_curated', 'format': 'tsv', 'action': 'Download' } url = 'http://ctdbase.org/tools/batchQuery.go?q' start = 0 end = min((batch_size, len(all_pubs))) # get them in batches of 4000 with open(disambig_file, 'wb') as f: while start < len(sorted_pubs): params['inputTerms'] = '|'.join(sorted_pubs[start:end]) # fetch the data from url logger.info( 'fetching %d (%d-%d) refs: %s', len(re.split(r'\|', params['inputTerms'])), start, end, params['inputTerms']) data = urllib.parse.urlencode(params) encoding = 'utf-8' binary_data = data.encode(encoding) req = urllib.request.Request(url, binary_data) resp = urllib.request.urlopen(req) f.write(resp.read()) start = end end = min((start + batch_size, len(sorted_pubs))) return def _process_interactions(self, row): """ Process row of CTD data from CTD_chemicals_diseases.tsv.gz and generate triples. Only create associations based on direct evidence (not using the inferred-via-gene), and unambiguous relationships. (Ambiguous ones will be processed in the sister method using the disambiguated file). There are no OMIM ids for diseases in these cases, so we associate with only the mesh disease ids. Args: :param row (list): row of CTD data Returns: :return None """ model = Model(self.g) self._check_list_len(row, 10) (chem_name, chem_id, cas_rn, disease_name, disease_id, direct_evidence, inferred_gene_symbol, inference_score, omim_ids, pubmed_ids) = row if direct_evidence == '': return evidence_pattern = re.compile(r'^therapeutic|marker\/mechanism$') # dual_evidence = re.compile(r'^marker\/mechanism\|therapeutic$') # filter on those diseases that are mapped to omim ids in the test set intersect = list( set(['OMIM:' + str(i) for i in omim_ids.split('|')] + [disease_id]) & set(self.test_diseaseids)) if self.testMode and len(intersect) < 1: return chem_id = 'MESH:' + chem_id reference_list = self._process_pubmed_ids(pubmed_ids) if re.match(evidence_pattern, direct_evidence): rel_id = self._get_relationship_id(direct_evidence) model.addClassToGraph(chem_id, chem_name) model.addClassToGraph(disease_id, None) self._make_association(chem_id, disease_id, rel_id, reference_list) else: # there's dual evidence, but haven't mapped the pubs pass # logger.debug( # "Dual evidence for %s (%s) and %s (%s)", # chem_name, chem_id, disease_name, disease_id) return def _process_disease2gene(self, row): """ Here, we process the disease-to-gene associations. Note that we ONLY process direct associations (not inferred through chemicals). Furthermore, we also ONLY process "marker/mechanism" associations. We preferentially utilize OMIM identifiers over MESH identifiers for disease/phenotype. Therefore, if a single OMIM id is listed under the "omim_ids" list, we will choose this over any MeSH id that might be listed as the disease_id. If multiple OMIM ids are listed in the omim_ids column, we toss this for now. (Mostly, we are not sure what to do with this information.) We associate "some variant of gene X" with the phenotype, rather than the gene directly. We also pull in the MeSH labels here (but not OMIM) to ensure that we have them (as they may not be brought in separately). :param row: :return: """ # if self.testMode: # g = self.testgraph # else: # g = self.graph # self._check_list_len(row, 9) # geno = Genotype(g) # gu = GraphUtils(curie_map.get()) model = Model(self.g) (gene_symbol, gene_id, disease_name, disease_id, direct_evidence, inference_chemical_name, inference_score, omim_ids, pubmed_ids) = row # we only want the direct associations; skipping inferred for now if direct_evidence == '' or direct_evidence != 'marker/mechanism': return # scrub some of the associations... # it seems odd to link human genes to the following "diseases" diseases_to_scrub = [ 'MESH:D004283', # dog diseases 'MESH:D004195', # disease models, animal 'MESH:D030342', # genetic diseases, inborn 'MESH:D040181', # genetic dieases, x-linked 'MESH:D020022'] # genetic predisposition to a disease if disease_id in diseases_to_scrub: logger.info( "Skipping association between NCBIGene:%s and %s", str(gene_id), disease_id) return intersect = list( set(['OMIM:' + str(i) for i in omim_ids.split('|')] + [disease_id]) & set(self.test_diseaseids)) if self.testMode and ( int(gene_id) not in self.test_geneids or len(intersect) < 1): return # there are three kinds of direct evidence: # (marker/mechanism | marker/mechanism|therapeutic | therapeutic) # we are only using the "marker/mechanism" for now # TODO what does it mean for a gene to be therapeutic for disease? # a therapeutic target? gene_id = 'NCBIGene:' + gene_id preferred_disease_id = disease_id if omim_ids is not None and omim_ids != '': omim_id_list = re.split(r'\|', omim_ids) # If there is only one OMIM ID for the Disease ID # or in the omim_ids list, # use the OMIM ID preferentially over any MeSH ID. if re.match(r'OMIM:.*', disease_id): if len(omim_id_list) > 1: # the disease ID is an OMIM ID and # there is more than one OMIM entry in omim_ids. # Currently no entries satisfy this condition pass elif disease_id != ('OMIM:' + omim_ids): # the disease ID is an OMIM ID and # there is only one non-equiv OMIM entry in omim_ids # we preferentially use the disease_id here logger.warning( "There may be alternate identifier for %s: %s", disease_id, omim_ids) # TODO: What should be done with the alternate disease IDs? else: if len(omim_id_list) == 1: # the disease ID is not an OMIM ID # and there is only one OMIM entry in omim_ids. preferred_disease_id = 'OMIM:' + omim_ids elif len(omim_id_list) > 1: # This is when the disease ID is not an OMIM ID and # there is more than one OMIM entry in omim_ids. pass # we actually want the association between the gene and the disease # to be via an alternate locus not the "wildtype" gene itself. So we # make an anonymous alternate locus, and put that in the association. alt_id = gene_id + '-' + preferred_disease_id + 'VL' # can't have colons in the bnodes alt_locus = re.sub(r':', '', alt_id) alt_locus = "_:" + alt_locus alt_label = 'some variant of ' + gene_symbol + ' that is ' \ + direct_evidence + ' for ' + disease_name model.addIndividualToGraph( alt_locus, alt_label, self.geno.genoparts['variant_locus']) # assume that the label gets added elsewhere model.addClassToGraph(gene_id, None) self.geno.addAffectedLocus(alt_locus, gene_id) model.addBlankNodeAnnotation(alt_locus) # not sure if MESH is getting added separately. # adding labels here for good measure dlabel = None if re.match(r'MESH', preferred_disease_id): dlabel = disease_name model.addClassToGraph(preferred_disease_id, dlabel) # Add the disease to gene relationship. rel_id = self._get_relationship_id(direct_evidence) refs = self._process_pubmed_ids(pubmed_ids) self._make_association(alt_locus, preferred_disease_id, rel_id, refs) return def _make_association(self, subject_id, object_id, rel_id, pubmed_ids): """ Make a reified association given an array of pubmed identifiers. Args: :param subject_id id of the subject of the association (gene/chem) :param object_id id of the object of the association (disease) :param rel_id relationship id :param pubmed_ids an array of pubmed identifiers Returns: :return None """ # TODO pass in the relevant Assoc class rather than relying on G2P assoc = G2PAssoc(self.g, self.name, subject_id, object_id, rel_id) if pubmed_ids is not None and len(pubmed_ids) > 0: eco = self._get_evidence_code('TAS') for pmid in pubmed_ids: r = Reference( self.g, pmid, Reference.ref_types['journal_article']) r.addRefToGraph() assoc.add_source(pmid) assoc.add_evidence(eco) assoc.add_association_to_graph() return @staticmethod def _process_pubmed_ids(pubmed_ids): """ Take a list of pubmed IDs and add PMID prefix Args: :param pubmed_ids - string representing publication ids seperated by a | symbol Returns: :return list: Pubmed curies """ if pubmed_ids.strip() == '': id_list = [] else: id_list = pubmed_ids.split('|') for (i, val) in enumerate(id_list): id_list[i] = 'PMID:' + val return id_list @staticmethod def _get_evidence_code(evidence): """ Get curie for evidence class label Args: :param evidence (str): evidence label Label: :return str: curie for evidence label from ECO """ eco_map = { 'TAS': 'ECO:0000033' } return eco_map[evidence] @staticmethod def _get_relationship_id(rel): """ Get curie from relationship property label Args: :param rel (str): relationship label Returns: :return str: curie for relationship label """ rel_map = { 'therapeutic': Model.object_properties['substance_that_treats'], 'marker/mechanism': Model.object_properties['is_marker_for'], } return str(rel_map[rel]) @staticmethod def _get_class_id(clslab): """ Get curie from CLASS_MAP dictionary Args: :param cls (str): class label Returns: :return str: curie for class label """ class_map = { 'pathway': 'PW:0000001', 'signal transduction': 'GO:0007165' } return class_map[clslab] def _parse_curated_chem_disease(self, limit): model = Model(self.g) line_counter = 0 file_path = '/'.join( (self.rawdir, self.static_files['publications']['file'])) with open(file_path, 'r') as tsvfile: reader = csv.reader(tsvfile, delimiter="\t") for row in reader: # catch comment lines if re.match(r'^#', ' '.join(row)): continue line_counter += 1 self._check_list_len(row, 10) (pub_id, disease_label, disease_id, disease_cat, evidence, chem_label, chem_id, cas_rn, gene_symbol, gene_acc) = row if disease_id.strip() == '' or chem_id.strip() == '': continue rel_id = self._get_relationship_id(evidence) chem_id = 'MESH:' + chem_id model.addClassToGraph(chem_id, chem_label) model.addClassToGraph(disease_id, None) if pub_id != '': pub_id = 'PMID:' + pub_id r = Reference( pub_id, Reference.ref_types['journal_article']) r.addRefToGraph(self.g) pubids = [pub_id] else: pubids = None self._make_association(chem_id, disease_id, rel_id, pubids) if not self.testMode and limit is not None \ and line_counter >= limit: break return def getTestSuite(self): import unittest from tests.test_ctd import CTDTestCase test_suite = unittest.TestLoader().loadTestsFromTestCase(CTDTestCase) # test_suite.addTests( # unittest.TestLoader().loadTestsFromTestCase(InteractionsTestCase)) return test_suite
def _process_allele_gene(self, limit): """ Make associations between an allele and a gene Adds triples to self.graph Approach is to use the label nomenclature and species map to determine taxon. Foreign Transgenes are filtered out. :param limit: number of rows to process :return: None """ geno = Genotype(self.graph) species_map = self._species_to_ncbi_tax() src_key = 'allele_gene' raw = '/'.join((self.rawdir, self.files[src_key]['file'])) LOG.info("processing allele to gene") col = self.files[src_key]['columns'] with gzip.open(raw, 'rt') as tsvfile: reader = csv.reader(tsvfile, delimiter='\t') # skip first line, version info next(reader) row = next(reader) # headers # header line starts with a hash and tab ?? row = row[1:] self.check_fileheader(col, row) for row in reader: allele_id = row[col.index('AlleleID')] allele_label = row[col.index('AlleleSymbol')] gene_id = row[col.index('GeneID')] gene_label = row[col.index('GeneSymbol')] allele_curie = 'FlyBase:' + allele_id gene_curie = 'FlyBase:' + gene_id # Add Allele and taxon, skip anything that's not drosophila allele_prefix = re.findall(r'^(\w*)\\', allele_label) if len(allele_prefix) == 1: try: if species_map[allele_prefix[0]][0] == 'drosophilid': geno.addAllele(allele_curie, allele_label) geno.addTaxon(species_map[allele_prefix[0]][1], allele_curie) else: # If it's a foreign transgenic allele, skip continue except KeyError: LOG.info("%s not in species prefix file", allele_prefix[0]) continue elif not allele_prefix: geno.addAllele(allele_curie, allele_label) geno.addTaxon(self.globaltt['Drosophila melanogaster'], allele_curie) else: raise ValueError( "Did not correctly parse allele label {}".format( allele_label)) # Process genes gene_prefix = re.findall(r'^(\w*)\\', gene_label) if len(gene_prefix) == 1: try: geno.addTaxon(species_map[gene_prefix[0]][1], gene_curie) if species_map[gene_prefix[0]][0] == 'drosophilid': geno.addGene(gene_curie, gene_label) else: # Don't create labels for non drosophila genes geno.addGene(gene_curie) except KeyError: LOG.info("%s not in species prefix file", gene_prefix[0]) geno.addGene(gene_curie) elif not gene_prefix: geno.addGene(gene_curie, gene_label) geno.addTaxon(self.globaltt['Drosophila melanogaster'], allele_curie) else: raise ValueError( "Did not correct parse gene label {}".format( gene_label)) # Connect allele and gene with geno.addAffectedLocus() if allele_prefix and gene_prefix: if allele_prefix[0] == gene_prefix[0]: geno.addAffectedLocus(allele_curie, gene_curie) else: raise ValueError( "Found allele and gene with different " "prefixes: {}, {}".format(allele_id, gene_id)) elif not allele_prefix and gene_prefix: raise ValueError("Found allele and gene with different " "prefixes: {}, {}".format( allele_id, gene_id)) else: # Both are melanogaster geno.addAffectedLocus(allele_curie, gene_curie) if limit is not None and reader.line_num > limit: break
def _process_diseasegene(self, limit): """ :param limit: :return: """ if self.testMode: g = self.testgraph else: g = self.graph line_counter = 0 geno = Genotype(g) model = Model(g) myfile = '/'.join((self.rawdir, self.files['disease-gene']['file'])) # PYLINT complains iterparse deprecated, # but as of py 3.4 only the optional & unsupplied parse arg is. for event, elem in ET.iterparse(myfile): if elem.tag == 'Disorder': # get the element name and id, ignoreS element name # id = elem.get('id') # some internal identifier disorder_num = elem.find('OrphaNumber').text disorder_id = 'Orphanet:' + str(disorder_num) if self.testMode and \ disorder_id not in \ config.get_config()['test_ids']['disease']: continue disorder_label = elem.find('Name').text # make a hash of internal gene id to type for later lookup gene_iid_to_type = {} gene_list = elem.find('GeneList') for gene in gene_list.findall('Gene'): gene_iid = gene.get('id') gene_type = gene.find('GeneType').get('id') gene_iid_to_type[gene_iid] = gene_type # assuming that these are in the ontology model.addClassToGraph(disorder_id, disorder_label) assoc_list = elem.find('DisorderGeneAssociationList') for a in assoc_list.findall('DisorderGeneAssociation'): gene_iid = a.find('.//Gene').get('id') gene_name = a.find('.//Gene/Name').text gene_symbol = a.find('.//Gene/Symbol').text gene_num = a.find('./Gene/OrphaNumber').text gene_id = 'Orphanet:' + str(gene_num) gene_type_id = \ self._map_gene_type_id(gene_iid_to_type[gene_iid]) model.addClassToGraph(gene_id, gene_symbol, gene_type_id, gene_name) syn_list = a.find('./Gene/SynonymList') if int(syn_list.get('count')) > 0: for s in syn_list.findall('./Synonym'): model.addSynonym(gene_id, s.text) dgtype = a.find('DisorderGeneAssociationType').get('id') rel_id = self._map_rel_id(dgtype) dg_label = \ a.find('./DisorderGeneAssociationType/Name').text if rel_id is None: logger.warning( "Cannot map association type (%s) to RO " + "for association (%s | %s). Skipping.", dg_label, disorder_label, gene_symbol) continue alt_locus_id = '_:' + gene_num + '-' + disorder_num + 'VL' alt_label = \ ' '.join(('some variant of', gene_symbol.strip(), 'that is a', dg_label.lower(), disorder_label)) model.addIndividualToGraph(alt_locus_id, alt_label, geno.genoparts['variant_locus']) geno.addAffectedLocus(alt_locus_id, gene_id) model.addBlankNodeAnnotation(alt_locus_id) # consider typing the gain/loss-of-function variants like: # http://sequenceontology.org/browser/current_svn/term/SO:0002054 # http://sequenceontology.org/browser/current_svn/term/SO:0002053 # use "assessed" status to issue an evidence code # FIXME I think that these codes are sub-optimal status_code = \ a.find('DisorderGeneAssociationStatus').get('id') # imported automatically asserted information # used in automatic assertion eco_id = 'ECO:0000323' # Assessed # TODO are these internal ids stable between releases? if status_code == '17991': # imported manually asserted information # used in automatic assertion eco_id = 'ECO:0000322' # Non-traceable author statement ECO_0000034 # imported information in automatic assertion ECO_0000313 assoc = G2PAssoc(g, self.name, alt_locus_id, disorder_id, rel_id) assoc.add_evidence(eco_id) assoc.add_association_to_graph() rlist = a.find('./Gene/ExternalReferenceList') eqid = None for r in rlist.findall('ExternalReference'): if r.find('Source').text == 'Ensembl': eqid = 'ENSEMBL:' + r.find('Reference').text elif r.find('Source').text == 'HGNC': eqid = 'HGNC:' + r.find('Reference').text elif r.find('Source').text == 'OMIM': eqid = 'OMIM:' + r.find('Reference').text else: pass # skip the others for now if eqid is not None: model.addClassToGraph(eqid, None) model.addEquivalentClass(gene_id, eqid) elem.clear() # empty the element if self.testMode and limit is not None and line_counter > limit: return return
def _process_qtls_genetic_location( self, raw, txid, common_name, limit=None): """ This function processes Triples created: :param limit: :return: """ if self.testMode: graph = self.testgraph else: graph = self.graph line_counter = 0 geno = Genotype(graph) model = Model(graph) eco_id = self.globaltt['quantitative trait analysis evidence'] taxon_curie = 'NCBITaxon:' + txid LOG.info("Processing genetic location for %s from %s", taxon_curie, raw) with open(raw, 'r', encoding="iso-8859-1") as csvfile: filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in filereader: line_counter += 1 (qtl_id, qtl_symbol, trait_name, assotype, empty, chromosome, position_cm, range_cm, flankmark_a2, flankmark_a1, peak_mark, flankmark_b1, flankmark_b2, exp_id, model_id, test_base, sig_level, lod_score, ls_mean, p_values, f_statistics, variance, bayes_value, likelihood_ratio, trait_id, dom_effect, add_effect, pubmed_id, gene_id, gene_id_src, gene_id_type, empty2) = row if self.testMode and int(qtl_id) not in self.test_ids: continue qtl_id = common_name + 'QTL:' + qtl_id.strip() trait_id = 'AQTLTrait:' + trait_id.strip() # Add QTL to graph feature = Feature(graph, qtl_id, qtl_symbol, self.globaltt['QTL']) feature.addTaxonToFeature(taxon_curie) # deal with the chromosome chrom_id = makeChromID(chromosome, taxon_curie, 'CHR') # add a version of the chromosome which is defined as # the genetic map build_id = 'MONARCH:'+common_name.strip()+'-linkage' build_label = common_name+' genetic map' geno.addReferenceGenome(build_id, build_label, taxon_curie) chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH') geno.addChromosomeInstance( chromosome, build_id, build_label, chrom_id) start = stop = None # range_cm sometimes ends in "(Mb)" (i.e pig 2016 Nov) range_mb = re.split(r'\(', range_cm) if range_mb is not None: range_cm = range_mb[0] if re.search(r'[0-9].*-.*[0-9]', range_cm): range_parts = re.split(r'-', range_cm) # check for poorly formed ranges if len(range_parts) == 2 and\ range_parts[0] != '' and range_parts[1] != '': (start, stop) = [ int(float(x.strip())) for x in re.split(r'-', range_cm)] else: LOG.info( "A cM range we can't handle for QTL %s: %s", qtl_id, range_cm) elif position_cm != '': match = re.match(r'([0-9]*\.[0-9]*)', position_cm) if match is not None: position_cm = match.group() start = stop = int(float(position_cm)) # FIXME remove converion to int for start/stop # when schema can handle floats add in the genetic location # based on the range feature.addFeatureStartLocation( start, chrom_in_build_id, None, [self.globaltt['FuzzyPosition']]) feature.addFeatureEndLocation( stop, chrom_in_build_id, None, [self.globaltt['FuzzyPosition']]) feature.addFeatureToGraph() # sometimes there's a peak marker, like a rsid. # we want to add that as a variant of the gene, # and xref it to the qtl. dbsnp_id = None if peak_mark != '' and peak_mark != '.' and \ re.match(r'rs', peak_mark.strip()): dbsnp_id = 'dbSNP:'+peak_mark.strip() model.addIndividualToGraph( dbsnp_id, None, self.globaltt['sequence_alteration']) model.addXref(qtl_id, dbsnp_id) gene_id = gene_id.replace('uncharacterized ', '').strip() if gene_id is not None and gene_id != '' and gene_id != '.'\ and re.fullmatch(r'[^ ]*', gene_id) is not None: # we assume if no src is provided and gene_id is an integer, # then it is an NCBI gene ... (okay, lets crank that back a notch) if gene_id_src == '' and gene_id.isdigit() and \ gene_id in self.gene_info: # LOG.info( # 'Warm & Fuzzy saying %s is a NCBI gene for %s', # gene_id, common_name) gene_id_src = 'NCBIgene' elif gene_id_src == '' and gene_id.isdigit(): LOG.warning( 'Cold & Prickely saying %s is a NCBI gene for %s', gene_id, common_name) gene_id_src = 'NCBIgene' elif gene_id_src == '': LOG.error( ' "%s" is a NOT NCBI gene for %s', gene_id, common_name) gene_id_src = None if gene_id_src == 'NCBIgene': gene_id = 'NCBIGene:' + gene_id # we will expect that these will get labels elsewhere geno.addGene(gene_id, None) # FIXME what is the right relationship here? geno.addAffectedLocus(qtl_id, gene_id) if dbsnp_id is not None: # add the rsid as a seq alt of the gene_id vl_id = '_:' + re.sub( r':', '', gene_id) + '-' + peak_mark.strip() geno.addSequenceAlterationToVariantLocus( dbsnp_id, vl_id) geno.addAffectedLocus(vl_id, gene_id) # add the trait model.addClassToGraph(trait_id, trait_name) # Add publication reference = None if re.match(r'ISU.*', pubmed_id): pub_id = 'AQTLPub:'+pubmed_id.strip() reference = Reference(graph, pub_id) elif pubmed_id != '': pub_id = 'PMID:' + pubmed_id.strip() reference = Reference( graph, pub_id, self.globaltt['journal article']) if reference is not None: reference.addRefToGraph() # make the association to the QTL assoc = G2PAssoc( graph, self.name, qtl_id, trait_id, self.globaltt['is marker for']) assoc.add_evidence(eco_id) assoc.add_source(pub_id) # create a description from the contents of the file # desc = '' # assoc.addDescription(g, assoc_id, desc) # TODO add exp_id as evidence # if exp_id != '': # exp_id = 'AQTLExp:'+exp_id # gu.addIndividualToGraph(g, exp_id, None, eco_id) if p_values != '': scr = re.sub(r'<', '', p_values) scr = re.sub(r',', '.', scr) # international notation if scr.isnumeric(): score = float(scr) assoc.set_score(score) # todo add score type # TODO add LOD score? assoc.add_association_to_graph() # make the association to the dbsnp_id, if found if dbsnp_id is not None: # make the association to the dbsnp_id assoc = G2PAssoc( graph, self.name, dbsnp_id, trait_id, self.globaltt['is marker for']) assoc.add_evidence(eco_id) assoc.add_source(pub_id) # create a description from the contents of the file # desc = '' # assoc.addDescription(g, assoc_id, desc) # TODO add exp_id # if exp_id != '': # exp_id = 'AQTLExp:'+exp_id # gu.addIndividualToGraph(g, exp_id, None, eco_id) if p_values != '': scr = re.sub(r'<', '', p_values) scr = re.sub(r',', '.', scr) if scr.isnumeric(): score = float(scr) assoc.set_score(score) # todo add score type # TODO add LOD score? assoc.add_association_to_graph() if not self.testMode and limit is not None and line_counter > limit: break LOG.info("Done with QTL genetic info") return
def _process_omim2gene(self, limit=None): """ This method maps the OMIM IDs and KEGG gene ID. Currently split based on the link_type field. Equivalent link types are mapped as gene XRefs. Reverse link types are mapped as disease to gene associations. Original link types are currently skipped. Triples created: <kegg_gene_id> is a Gene <omim_gene_id> is a Gene <kegg_gene_id>> hasXref <omim_gene_id> <assoc_id> has subject <omim_disease_id> <assoc_id> has object <kegg_gene_id> :param limit: :return: """ LOG.info("Processing OMIM to KEGG gene") if self.test_mode: graph = self.testgraph else: graph = self.graph model = Model(graph) geno = Genotype(graph) raw = '/'.join((self.rawdir, self.files['omim2gene']['file'])) with open(raw, 'r', encoding="iso-8859-1") as csvfile: reader = csv.reader(csvfile, delimiter='\t', quotechar='\"') for row in reader: (kegg_gene_id, omim_id, link_type) = row if self.test_mode and kegg_gene_id not in self.test_ids['genes']: continue kegg_gene_id = 'KEGG-' + kegg_gene_id.strip() omim_id = re.sub(r'omim', 'OMIM', omim_id) if link_type == 'equivalent': # these are genes! # so add them as a class then make equivalence model.addClassToGraph(omim_id, None) geno.addGene(kegg_gene_id, None) # previous: if omim type is not disease-ish then use # now is: if omim type is gene then use if omim_id in self.omim_replaced: repl = self.omim_replaced[omim_id] for omim in repl: if omim in self.omim_type and \ self.omim_type[omim] == self.globaltt['gene']: omim_id = omim if omim_id in self.omim_type and \ self.omim_type[omim_id] == self.globaltt['gene']: model.addEquivalentClass(kegg_gene_id, omim_id) elif link_type == 'reverse': # make an association between an OMIM ID & the KEGG gene ID # we do this with omim ids because # they are more atomic than KEGG ids alt_locus_id = self._make_variant_locus_id(kegg_gene_id, omim_id) alt_label = self.label_hash[alt_locus_id] model.addIndividualToGraph( alt_locus_id, alt_label, self.globaltt['variant_locus']) geno.addAffectedLocus(alt_locus_id, kegg_gene_id) model.addBlankNodeAnnotation(alt_locus_id) # Add the disease to gene relationship. rel = self.globaltt['is marker for'] assoc = G2PAssoc(graph, self.name, alt_locus_id, omim_id, rel) assoc.add_association_to_graph() elif link_type == 'original': # these are sometimes a gene, and sometimes a disease LOG.info( 'Unable to handle original link for %s-%s', kegg_gene_id, omim_id) else: # don't know what these are LOG.warning( 'Unhandled link type for %s-%s: %s', kegg_gene_id, omim_id, link_type) if (not self.test_mode) and ( limit is not None and reader.line_num > limit): break LOG.info("Done with OMIM to KEGG gene")
def _build_gene_disease_model( self, gene_id, relation_id, disease_id, variant_label, consequence_predicate=None, consequence_id=None, allelic_requirement=None, pmids=None): """ Builds gene variant disease model :return: None """ model = Model(self.graph) geno = Genotype(self.graph) pmids = [] if pmids is None else pmids is_variant = False variant_or_gene = gene_id variant_id_string = variant_label variant_bnode = self.make_id(variant_id_string, "_") if consequence_predicate is not None \ and consequence_id is not None: is_variant = True model.addTriple(variant_bnode, consequence_predicate, consequence_id) # Hack to add labels to terms that # don't exist in an ontology if consequence_id.startswith(':'): model.addLabel(consequence_id, consequence_id.strip(':').replace('_', ' ')) if is_variant: variant_or_gene = variant_bnode # Typically we would type the variant using the # molecular consequence, but these are not specific # enough for us to make mappings (see translation table) model.addIndividualToGraph(variant_bnode, variant_label, self.globaltt['variant_locus']) geno.addAffectedLocus(variant_bnode, gene_id) model.addBlankNodeAnnotation(variant_bnode) assoc = G2PAssoc( self.graph, self.name, variant_or_gene, disease_id, relation_id) assoc.source = pmids assoc.add_association_to_graph() if allelic_requirement is not None and is_variant is False: model.addTriple( assoc.assoc_id, self.globaltt['has_allelic_requirement'], allelic_requirement) if allelic_requirement.startswith(':'): model.addLabel( allelic_requirement, allelic_requirement.strip(':').replace('_', ' '))
def _process_haplotype( self, hap_id, hap_label, chrom_num, chrom_pos, context, risk_allele_frequency, mapped_gene, so_ontology): tax_id = 'NCBITaxon:9606' if self.testMode: g = self.testgraph else: g = self.graph geno = Genotype(g) model = Model(g) # add the feature to the graph hap_description = None if risk_allele_frequency != '' and \ risk_allele_frequency != 'NR': hap_description = \ str(risk_allele_frequency) + \ ' [risk allele frequency]' model.addIndividualToGraph(hap_id, hap_label.strip(), Feature.types['haplotype'], hap_description) geno.addTaxon(tax_id, hap_id) snp_labels = re.split(r';\s?', hap_label) chrom_nums = re.split(r';\s?', chrom_num) chrom_positions = re.split(r';\s?', chrom_pos) context_list = re.split(r';\s?', context) mapped_genes = re.split(r';\s?', mapped_gene) snp_curies = list() for index, snp in enumerate(snp_labels): snp_curie, snp_type = self._get_curie_and_type_from_id(snp) if snp_type is None: # make blank node snp_curie = self.make_id(snp, "_") g.addTriple(hap_id, geno.object_properties['has_variant_part'], snp_curie) snp_curies.append(snp_curie) # courtesy http://stackoverflow.com/a/16720915 length = len(snp_labels) if not all(len(lst) == length for lst in [chrom_nums, chrom_positions, context_list]): logger.warn( "Unexpected data field for haplotype {} \n " "will not add snp details".format(hap_label)) return variant_in_gene_count = 0 for index, snp_curie in enumerate(snp_curies): self._add_snp_to_graph( snp_curie, snp_labels[index], chrom_nums[index], chrom_positions[index], context_list[index]) if len(mapped_genes) == len(snp_labels): so_class = self._map_variant_type(context_list[index]) if so_class is None: raise ValueError("Unknown SO class {} in haplotype {}" .format(context_list[index], hap_label)) so_query = """ SELECT ?variant_label WHERE {{ {0} rdfs:subClassOf+ SO:0001564 ; rdfs:label ?variant_label . }} """.format(so_class) query_result = so_ontology.query(so_query) if len(list(query_result)) > 0: gene_id = DipperUtil.get_ncbi_id_from_symbol( mapped_genes[index]) if gene_id is not None: geno.addAffectedLocus(snp_curie, gene_id) geno.addAffectedLocus(hap_id, gene_id) variant_in_gene_count += 1 if context_list[index] == 'upstream_gene_variant': gene_id = DipperUtil.get_ncbi_id_from_symbol( mapped_genes[index]) if gene_id is not None: g.addTriple( snp_curie, Feature.object_properties[ 'upstream_of_sequence_of'], gene_id) elif context_list[index] == 'downstream_gene_variant': gene_id = DipperUtil.get_ncbi_id_from_symbol( mapped_genes[index]) if gene_id is not None: g.addTriple( snp_curie, Feature.object_properties[ 'downstream_of_sequence_of'], gene_id) else: logger.warn("More mapped genes than snps, " "cannot disambiguate for {}".format(hap_label)) # Seperate in case we want to apply a different relation # If not this is redundant with triples added above if len(mapped_genes) == variant_in_gene_count \ and len(set(mapped_genes)) == 1: gene_id = DipperUtil.get_ncbi_id_from_symbol(mapped_genes[0]) geno.addAffectedLocus(hap_id, gene_id) return
def _process_allele_gene(self, limit): """ Make associations between an allele and a gene Adds triples to self.graph Approach is to use the label nomenclature and species map to determine taxon. Foreign Transgenes are filtered out. :param limit: number of rows to process :return: None """ geno = Genotype(self.graph) species_map = self._species_to_ncbi_tax() src_key = 'allele_gene' raw = '/'.join((self.rawdir, self.files[src_key]['file'])) LOG.info("processing allele to gene") col = self.files[src_key]['columns'] with gzip.open(raw, 'rt') as tsvfile: reader = csv.reader(tsvfile, delimiter='\t') # skip first line, version info next(reader) row = next(reader) # headers # header line starts with a hash and tab ?? row = row[1:] self.check_fileheader(col, row) for row in reader: allele_id = row[col.index('AlleleID')] allele_label = row[col.index('AlleleSymbol')] gene_id = row[col.index('GeneID')] gene_label = row[col.index('GeneSymbol')] allele_curie = 'FlyBase:' + allele_id gene_curie = 'FlyBase:' + gene_id # Add Allele and taxon, skip anything that's not drosophila allele_prefix = re.findall(r'^(\w*)\\', allele_label) if len(allele_prefix) == 1: try: if species_map[allele_prefix[0]][0] == 'drosophilid': geno.addAllele(allele_curie, allele_label) geno.addTaxon(species_map[allele_prefix[0]][1], allele_curie) else: # If it's a foreign transgenic allele, skip continue except KeyError: LOG.info("%s not in species prefix file", allele_prefix[0]) note = ''' list of unincluded species prefixes include: Aace,Afun,Agos,Ahyp,Amil,Aobl,Apim,Apol,Aque,Asam,AspBV3L6, Avin,Baen,Bant,Bcen,Bdor,Beme,Besp,Bger,Blan,Bovi,Brsp, Bsp240B1,Bsub,Btab,Bter,Bxb1,BYV,CABYV,Cbeta,Ccaj,Cdif, Cfum,Cgri,Cint,Clsp,Cmar,Cnoc,Cpip,Cprd,Cqui,Crub,Csal, CsIV,D6,Dano,Dcaa,Dcol,Dcub,Ddun,DENV,Dflo,Dful,Dmas,Dnep, Drad,Ecab,Efae,Egra,Epos,Equa,EspSC22,Fmer,Gfas,Gint,Gmax, Gmor,Gthe,gypsy,Harm,hobo,HPV18,Hpyl,Hsod,HspTP009,Htur, Hver,Isca,jockey,Klac,Kpne,Lcup,Ldis,Lhem,Lmal,Lmon,Lser, Mani,Mbre,Mosp,Mper,Mril,NDV,Nlug,Npha,Nvec,Nvit,Oari, Obic,Osat,Paer,Pchi,PCV,Penelope,Pgur,Phum,Pime,Pmat,Pshi, Pvin,PVX,Pxyl,Rfla,Rhsp,Rpal,Rsph,Shel,Slit,Soce,Spou, Spyo,Tadh,TBSV,TCV,TEV,Tgeo,Tgon,Tmer,TNPV,TspX513,Tthe, Vcon,Vdes,Vpar,VV,WSSV,Xvas,Zbai,Zbis,ZIKV,Zrou,ZYMV ''' continue elif not allele_prefix: geno.addAllele(allele_curie, allele_label) geno.addTaxon(self.globaltt['Drosophila melanogaster'], allele_curie) else: raise ValueError( "Did not correctly parse allele label {}".format( allele_label)) # Process genes gene_prefix = re.findall(r'^(\w*)\\', gene_label) if len(gene_prefix) == 1: try: geno.addTaxon(species_map[gene_prefix[0]][1], gene_curie) if species_map[gene_prefix[0]][0] == 'drosophilid': geno.addGene(gene_curie, gene_label) else: # Don't create labels for non drosophila genes geno.addGene(gene_curie) except KeyError: LOG.info("%s not in species prefix file", gene_prefix[0]) geno.addGene(gene_curie) elif not gene_prefix: geno.addGene(gene_curie, gene_label) geno.addTaxon(self.globaltt['Drosophila melanogaster'], allele_curie) else: raise ValueError( "Did not correct parse gene label {}".format( gene_label)) # Connect allele and gene with geno.addAffectedLocus() if allele_prefix and gene_prefix: if allele_prefix[0] == gene_prefix[0]: geno.addAffectedLocus(allele_curie, gene_curie) else: raise ValueError( "Found allele and gene with different " "prefixes: {}, {}".format(allele_id, gene_id)) elif not allele_prefix and gene_prefix: raise ValueError("Found allele and gene with different " "prefixes: {}, {}".format( allele_id, gene_id)) else: # Both are melanogaster geno.addAffectedLocus(allele_curie, gene_curie) if limit is not None and reader.line_num > limit: break
def _process_haplotype( self, hap_id, hap_label, chrom_num, chrom_pos, context, risk_allele_frequency, mapped_gene, so_ontology): if self.test_mode: graph = self.testgraph else: graph = self.graph geno = Genotype(graph) model = Model(graph) # add the feature to the graph hap_description = None if risk_allele_frequency != '' and risk_allele_frequency != 'NR': hap_description = str(risk_allele_frequency) + ' [risk allele frequency]' model.addIndividualToGraph( hap_id, hap_label.strip(), self.globaltt['haplotype'], hap_description) geno.addTaxon(self.globaltt["H**o sapiens"], hap_id) snp_labels = re.split(r';\s?', hap_label) chrom_nums = re.split(r';\s?', chrom_num) chrom_positions = re.split(r';\s?', chrom_pos) context_list = re.split(r';\s?', context) mapped_genes = re.split(r';\s?', mapped_gene) snp_curies = list() for index, snp in enumerate(snp_labels): snp_curie, snp_type = self._get_curie_and_type_from_id(snp) if snp_type is None: # make blank node snp_curie = self.make_id(snp, "_") graph.addTriple(hap_id, self.globaltt['has_variant_part'], snp_curie) snp_curies.append(snp_curie) # courtesy http://stackoverflow.com/a/16720915 length = len(snp_labels) if not all(len(lst) == length for lst in [chrom_nums, chrom_positions, context_list]): LOG.warning( "Unexpected data field for haplotype %s \n " "will not add snp details", hap_label) return variant_in_gene_count = 0 for index, snp_curie in enumerate(snp_curies): self._add_snp_to_graph( snp_curie, snp_labels[index], chrom_nums[index], chrom_positions[index], context_list[index]) if len(mapped_genes) == len(snp_labels): so_class = self.resolve(context_list[index]) # removed the '+' for recursive one-or-more rdfs:subClassOf paths # just so it did not return an empty graph so_query = """ SELECT ?variant_label WHERE {{ {0} rdfs:subClassOf {1} ; rdfs:label ?variant_label . }} """.format(so_class, self.globaltt['gene_variant']) query_result = so_ontology.query(so_query) if len(list(query_result)) == 1: gene_id = DipperUtil.get_ncbi_id_from_symbol(mapped_genes[index]) if gene_id is not None: geno.addAffectedLocus(snp_curie, gene_id) geno.addAffectedLocus(hap_id, gene_id) variant_in_gene_count += 1 gene_id = DipperUtil.get_ncbi_id_from_symbol(mapped_genes[index]) if gene_id is not None: graph.addTriple( snp_curie, self.resolve(context_list[index]), gene_id) else: LOG.warning( "More mapped genes than snps, cannot disambiguate for %s", hap_label) # Seperate in case we want to apply a different relation # If not this is redundant with triples added above if len(mapped_genes) == variant_in_gene_count and len(set(mapped_genes)) == 1: gene_id = DipperUtil.get_ncbi_id_from_symbol(mapped_genes[0]) geno.addAffectedLocus(hap_id, gene_id) return
def _process_haplotype(self, hap_id, hap_label, chrom_num, chrom_pos, context, risk_allele_frequency, mapped_gene, so_ontology): if self.test_mode: graph = self.testgraph else: graph = self.graph geno = Genotype(graph) model = Model(graph) # add the feature to the graph hap_description = None if risk_allele_frequency not in ['', 'NR']: hap_description = str( risk_allele_frequency) + ' [risk allele frequency]' model.addIndividualToGraph(hap_id, hap_label.strip(), self.globaltt['haplotype'], hap_description) geno.addTaxon(self.globaltt["H**o sapiens"], hap_id) snp_labels = re.split(r';\s?', hap_label) chrom_nums = re.split(r';\s?', chrom_num) chrom_positions = re.split(r';\s?', chrom_pos) context_list = re.split(r';\s?', context) mapped_genes = re.split(r';\s?', mapped_gene) # Not having four "PAX5" as a list might be better, but it breaks unit tests # mapped_genes = list(set(mapped_genes)) # make uniq # snp_labels = list(set(snp_labels)) # make uniq snp_curies = list() for snp in snp_labels: snp_curie, snp_type = self._get_curie_and_type_from_id(snp) if snp_type is None: LOG.info('cant find type for SNP in %s', snp) # make blank node snp_curie = self.make_id(snp, "_") model.addLabel(snp_curie, snp) elif snp_curie[0] == '_': # arrived an unlabeled blanknode model.addLabel(snp_curie, snp) graph.addTriple(hap_id, self.globaltt['has_variant_part'], snp_curie) snp_curies.append(snp_curie) # courtesy http://stackoverflow.com/a/16720915 # check lengths of mutiple lists length = len(snp_curies) if not all( len(lst) == length for lst in [snp_labels, chrom_nums, chrom_positions, context_list]): LOG.warning( "Incongruous data field(s) for haplotype %s \n " "will not add snp details", hap_label) else: variant_in_gene_count = 0 for index, snp_curie in enumerate(snp_curies): self._add_snp_to_graph(snp_curie, snp_labels[index], chrom_nums[index], chrom_positions[index], context_list[index]) if mapped_genes and len(mapped_genes) != len(snp_labels): LOG.warning("More mapped genes than snps," " cannot disambiguate for\n%s\n%s", mapped_genes, snp_labels) # hap_label) else: so_class = self.resolve(context_list[index]) so_query = """ SELECT ?variant_label WHERE {{ {0} rdfs:subClassOf+ {1} ; rdfs:label ?variant_label . }} """.format(so_class, self.globaltt['gene_variant']) query_result = so_ontology.query(so_query) gene_id = DipperUtil.get_hgnc_id_from_symbol( mapped_genes[index]) if gene_id is not None and len(list(query_result)) == 1: if context_list[index] in [ 'upstream_gene_variant', 'downstream_gene_variant' ]: graph.addTriple(snp_curie, self.resolve(context_list[index]), gene_id) else: geno.addAffectedLocus(snp_curie, gene_id) variant_in_gene_count += 1 # Seperate in case we want to apply a different relation # If not this is redundant with triples added above if len(mapped_genes) == variant_in_gene_count and \ len(set(mapped_genes)) == 1: gene_id = DipperUtil.get_hgnc_id_from_symbol(mapped_genes[0]) geno.addAffectedLocus(hap_id, gene_id)
def add_gene_to_disease(self, association_type, gene_id, gene_symbol, disease_id, eco_id): """ Composes triples based on the DisorderGeneAssociationType element: AND the suffixes: - "gene phenotype" - "function consequence" - "cell origin" xmlstarlet sel -t -v "/JDBOR/DisorderList/Disorder/DisorderGeneAssociationList/ DisorderGeneAssociation/DisorderGeneAssociationType/Name" en_product6.xml \ | sort -u Biomarker tested in Candidate gene tested in Disease-causing germline mutation(s) (gain of function) in Disease-causing germline mutation(s) in Disease-causing germline mutation(s) (loss of function) in Disease-causing somatic mutation(s) in Major susceptibility factor in Modifying germline mutation in Part of a fusion gene in Role in the phenotype of These labels are a composition of terms, we map: gene-disease predicate (has phenotype, contributes_to) variant-origin (germline, somatic) variant-functional consequence (loss, gain) To check on the "DisorderGeneAssociationType" to id-label map xmlstarlet sel -t -m \ './JDBOR/DisorderList/Disorder/DisorderGeneAssociationList/\ DisorderGeneAssociation/DisorderGeneAssociationType'\ -v './@id' -o ' ' -v './Name' -n en_product6.xml |\ sort | uniq -c | sort -nr Although the id-label pairs appear to be stable after a few years, we map to the label instead of the id in case Orphanet changes their IDs :param association_type: {str} DisorderGeneAssociationType/Name, eg Role in the phenotype of :param gene_id: {str} gene id as curie :param gene_symbol: {str} HGVS gene symbol :param disease_id: {str} disease id as curie :param eco_id: {str} eco code as curie :return: None """ model = Model(self.graph) geno = Genotype(self.graph) gene_or_variant = "" # If we know something about the variant such as functional consequence or # cellular origin make a blank node and attach the attributes is_variant = False variant_id_string = "{}{}".format(gene_id, disease_id) functional_consequence = None cell_origin = None # hard fail for no mappings/new terms, otherwise they go unnoticed if "{}|gene phenotype".format(association_type) not in self.localtt: raise ValueError( 'Disease-gene association type {} not mapped'.format( association_type)) g2p_relation = self.resolve("|".join( [association_type, "gene phenotype"])) # Variant attributes if "|".join([association_type, "function consequence"]) in self.localtt: is_variant = True local_key = "|".join([association_type, "function consequence"]) functional_consequence = self.resolve(local_key) functional_consequence_lbl = self.localtt[local_key] if "|".join([association_type, "cell origin"]) in self.localtt: is_variant = True local_key = "|".join([association_type, "cell origin"]) cell_origin = self.resolve(local_key) cell_origin_lbl = self.localtt[local_key] if is_variant: variant_label = "of {}".format(gene_symbol) if functional_consequence: variant_label = "{} {}".format( functional_consequence_lbl.replace('_', ' '), variant_label) variant_id_string += functional_consequence_lbl else: variant_label = "variant {}".format(variant_label) if cell_origin: variant_label = "{} {}".format(cell_origin_lbl, variant_label) variant_id_string += cell_origin_lbl variant_bnode = self.make_id(variant_id_string, "_") model.addIndividualToGraph(variant_bnode, variant_label, self.globaltt['variant_locus']) geno.addAffectedLocus(variant_bnode, gene_id) model.addBlankNodeAnnotation(variant_bnode) self._add_variant_attributes(variant_bnode, functional_consequence, cell_origin) gene_or_variant = variant_bnode else: gene_or_variant = gene_id assoc = G2PAssoc(self.graph, self.name, gene_or_variant, disease_id, g2p_relation) assoc.add_evidence(eco_id) assoc.add_association_to_graph() return