def _process_phene_gene_row(self, row):
geno = Genotype(self.graph)
model = Model(self.graph)
gene_id = self.id_hash['gene'].get(row['gene_id'])
phene_id = self.id_hash['phene'].get(row['phene_id'])
omia_id = self._get_omia_id_from_phene_id(phene_id)
if self.test_mode and not (omia_id in self.test_ids['disease']
and row['gene_id'] in self.test_ids['gene']
) or gene_id is None or phene_id is None:
return
# occasionally some phenes are missing! (ex: 406)
if phene_id is None:
LOG.warning("Phene id %s is missing", str(row['phene_id']))
return
gene_label = self.label_hash[gene_id]
# some variant of gene_id has phenotype d
var = self.make_id(gene_id.split(':')[-1] + 'VL', '_')
geno.addAllele(var, 'some variant of ' + gene_label)
geno.addAlleleOfGene(var, gene_id)
geno.addAffectedLocus(var, gene_id)
model.addBlankNodeAnnotation(var)
assoc = G2PAssoc(self.graph, self.name, var, phene_id)
assoc.add_association_to_graph()
# add the gene id to the set of annotated genes
# for later lookup by orthology
self.annotated_genes.add(gene_id)
def _process_phene_gene_row(self, row):
geno = Genotype(self.g)
model = Model(self.g)
gene_id = self.id_hash['gene'].get(row['gene_id'])
phene_id = self.id_hash['phene'].get(row['phene_id'])
omia_id = self._get_omia_id_from_phene_id(phene_id)
if self.testMode and not (
omia_id in self.test_ids['disease'] and
row['gene_id'] in self.test_ids['gene']) or\
gene_id is None or phene_id is None:
return
# occasionally some phenes are missing! (ex: 406)
if phene_id is None:
logger.warning("Phene id %s is missing", str(row['phene_id']))
return
gene_label = self.label_hash[gene_id]
# some variant of gene_id has phenotype d
vl = '_:'+re.sub(r'NCBIGene:', '', str(gene_id)) + 'VL'
geno.addAllele(vl, 'some variant of ' + gene_label)
geno.addAlleleOfGene(vl, gene_id)
geno.addAffectedLocus(vl, gene_id)
model.addBlankNodeAnnotation(vl)
assoc = G2PAssoc(self.g, self.name, vl, phene_id)
assoc.add_association_to_graph()
# add the gene id to the set of annotated genes
# for later lookup by orthology
self.annotated_genes.add(gene_id)
return
def process_disease_association(self, limit):
raw = '/'.join((self.rawdir, self.files['disease_assoc']['file']))
if self.testMode:
g = self.testgraph
else:
g = self.graph
gu = GraphUtils(curie_map.get())
logger.info("Processing disease models")
geno = Genotype(g, self.nobnodes)
line_counter = 0
worm_taxon = 'NCBITaxon:6239'
with open(raw, 'r') as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
if re.match(r'!', ''.join(row)): # header
continue
line_counter += 1
(db, gene_num, gene_symbol, is_not, disease_id, ref,
eco_symbol, with_or_from, aspect, gene_name, gene_synonym,
gene_class, taxon, date, assigned_by, blank, blank2) = row
if self.testMode and gene_num not in self.test_ids['gene']:
continue
# TODO add NOT phenotypes
if is_not == 'NOT':
continue
# WB WBGene00000001 aap-1 DOID:2583 PMID:19029536 IEA ENSEMBL:ENSG00000145675|OMIM:615214 D Y110A7A.10 gene taxon:6239 20150612 WB
gene_id = 'WormBase:'+gene_num
# make a variant of the gene
vl = '_'+'-'.join((gene_num, 'unspecified'))
if self.nobnodes:
vl = ':'+vl
vl_label = 'some variant of '+gene_symbol
geno.addAlleleOfGene(vl, gene_id)
animal_id = geno.make_experimental_model_with_genotype(
g, vl, vl_label, worm_taxon, 'worm')
assoc = G2PAssoc(
self.name, animal_id,
disease_id, gu.object_properties['model_of'])
ref = re.sub(r'WB_REF:', 'WormBase:', ref)
if ref != '':
assoc.add_source(ref)
eco_id = None
if eco_symbol == 'IEA':
eco_id = 'ECO:0000501' # IEA is this now
if eco_id is not None:
assoc.add_evidence(eco_id)
assoc.add_association_to_graph(g)
return
def _get_process_allelic_variants(self, entry, g):
gu = GraphUtils(curie_map.get())
geno = Genotype(g)
du = DipperUtil()
if entry is not None:
publist = {} # to hold the entry-specific publication mentions for the allelic variants
entry_num = entry['mimNumber']
# process the ref list just to get the pmids
ref_to_pmid = self._get_pubs(entry, g)
if 'allelicVariantList' in entry:
allelicVariantList = entry['allelicVariantList']
for al in allelicVariantList:
al_num = al['allelicVariant']['number']
al_id = 'OMIM:'+str(entry_num)+'.'+str(al_num).zfill(4)
al_label = None
al_description = None
if al['allelicVariant']['status'] == 'live':
publist[al_id] = set()
if 'mutations' in al['allelicVariant']:
al_label = al['allelicVariant']['mutations']
if 'text' in al['allelicVariant']:
al_description = al['allelicVariant']['text']
m = re.findall('\{(\d+)\:', al_description)
publist[al_id] = set(m)
geno.addAllele(al_id, al_label, geno.genoparts['variant_locus'], al_description)
geno.addAlleleOfGene(al_id, 'OMIM:'+str(entry_num),
geno.object_properties['is_sequence_variant_instance_of'])
for r in publist[al_id]:
pmid = ref_to_pmid[int(r)]
gu.addTriple(g, pmid, gu.object_properties['is_about'], al_id)
# look up the pubmed id in the list of references
if 'dbSnps' in al['allelicVariant']:
dbsnp_ids = re.split(',', al['allelicVariant']['dbSnps'])
for dnum in dbsnp_ids:
did = 'dbSNP:'+dnum.strip()
gu.addIndividualToGraph(g, did, None)
gu.addEquivalentClass(g, al_id, did)
if 'clinvarAccessions' in al['allelicVariant']:
# clinvarAccessions triple semicolon delimited, each lik eRCV000020059;;1
rcv_ids = re.split(';;;', al['allelicVariant']['clinvarAccessions'])
rcv_ids = [(re.match('(RCV\d+)\;\;', r)).group(1) for r in rcv_ids]
for rnum in rcv_ids:
rid = 'ClinVar:'+rnum
gu.addXref(g, al_id, rid)
gu.addPage(g, al_id, "http://omim.org/entry/"+str(entry_num)+"#"+str(al_num).zfill(4))
elif re.search('moved', al['allelicVariant']['status']):
# for both 'moved' and 'removed'
moved_ids = None
if 'movedTo' in al['allelicVariant']:
moved_id = 'OMIM:'+al['allelicVariant']['movedTo']
moved_ids = [moved_id]
gu.addDeprecatedIndividual(g, al_id, moved_ids)
else:
logger.error('Uncaught alleleic variant status %s', al['allelicVariant']['status'])
# end loop allelicVariantList
return
def _add_variant_gene_relationship(self, variant_id, hgnc_symbol):
"""
:param variant_id
:param hgnc_symbol
:return: None
"""
gu = GraphUtils(curie_map.get())
geno = Genotype(self.graph)
if hgnc_symbol in self.gene_map:
gene_id = self.gene_map[hgnc_symbol]
else:
gene_id = self.make_cgd_id("{0}{1}".format(variant_id, hgnc_symbol))
logger.warn("Can't map gene symbol {0} "
"to entrez ID".format(hgnc_symbol))
gu.addClassToGraph(self.graph, gene_id, hgnc_symbol)
geno.addAlleleOfGene(variant_id, gene_id)
return
def process_allele_phenotype(self, limit=None):
"""
This file compactly lists variant to phenotype associations,
such that in a single row, there may be >1 variant listed
per phenotype and paper. This indicates that each variant is
individually assocated with the given phenotype,
as listed in 1+ papers.
(Not that the combination of variants is producing the phenotype.)
:param limit:
:return:
"""
raw = '/'.join((self.rawdir, self.files['allele_pheno']['file']))
graph = self.graph
model = Model(self.graph)
LOG.info("Processing Allele phenotype associations")
line_counter = 0
geno = Genotype(graph)
with open(raw, 'r') as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
if re.match(r'!', ''.join(row)): # header
continue
line_counter += 1
(db, gene_num, gene_symbol, is_not, phenotype_id, ref,
eco_symbol, with_or_from, aspect, gene_name, gene_synonym,
gene_class, taxon, date, assigned_by, blank, blank2) = row
# TODO add NOT phenotypes
if is_not == 'NOT':
continue
eco_symbol = eco_symbol.strip()
eco_id = None
if eco_symbol.strip() != '':
eco_id = self.resolve(eco_symbol)
# according to the GOA spec, persons are not allowed to be
# in the reference column, therefore they the variant and
# persons are swapped between the reference and with column.
# we unswitch them here.
temp_var = temp_ref = None
if re.search(r'WBVar|WBRNAi', ref):
temp_var = ref
# move the paper from the with column into the ref
if re.search(r'WBPerson', with_or_from):
temp_ref = with_or_from
if temp_var is not None or temp_ref is not None:
with_or_from = temp_var
ref = temp_ref
allele_list = re.split(r'\|', with_or_from)
if len(allele_list) == 0:
LOG.error(
"Missing alleles from phenotype assoc at line %d",
line_counter)
continue
else:
for allele in allele_list:
allele_num = re.sub(r'WB:', '', allele.strip())
allele_id = 'WormBase:' + allele_num
gene_id = 'WormBase:' + gene_num
if re.search(r'WBRNAi', allele_id):
# @kshefchek - removing this blank node
# in favor of simpler modeling
# make the WormBase:WBRNAi* id
# a self.globaltt['reagent_targeted_gene'], and attach
# phenotype to this ID
# Previous model - make a bnode reagent-targeted gene,
# & annotate that instead of the RNAi item directly
#rnai_num = re.sub(r'WormBase:', '', allele_id)
#rnai_id = allele_id
#rtg_id = self.make_reagent_targeted_gene_id(
# gene_num, rnai_num)
#geno.addReagentTargetedGene(
# rnai_id, 'WormBase:' + gene_num, rtg_id)
# allele_id = rtg_id
# Could type the IRI as both the reagant and reagant
# targeted gene but not sure if this needed
# geno.addGeneTargetingReagent(
# allele_id, None, self.globaltt['RNAi_reagent'], gene_id)
model.addIndividualToGraph(
allele_id, None,
self.globaltt['reagent_targeted_gene'])
self.graph.addTriple(
allele_id, self.globaltt['is_expression_variant_of'],
gene_id)
elif re.search(r'WBVar', allele_id):
# this may become deprecated by using wormmine
# make the allele to gene relationship
# the WBVars are really sequence alterations
# the public name will come from elsewhere
# @kshefchek - removing this blank node
# in favor of simpler modeling, treat variant
# like an allele
#vl_id = '_:'+'-'.join((gene_num, allele_num))
#geno.addSequenceAlterationToVariantLocus(
# allele_id, vl_id)
#geno.addAlleleOfGene(vl_id, gene_id)
geno.addSequenceAlteration(allele_id, None)
geno.addAlleleOfGene(allele_id, gene_id)
else:
LOG.warning(
"Some kind of allele I don't recognize: %s", allele_num)
continue
assoc = G2PAssoc(graph, self.name, allele_id, phenotype_id)
if eco_id is not None:
assoc.add_evidence(eco_id)
if ref is not None and ref != '':
ref = re.sub(r'(WB:|WB_REF:)', 'WormBase:', ref)
reference = Reference(graph, ref)
if re.search(r'Person', ref):
reference.setType(self.globaltt['person'])
assoc.add_evidence(
self.globaltt[
'inference from background scientific knowledge'
])
reference.addRefToGraph()
assoc.add_source(ref)
assoc.add_association_to_graph()
# finish looping through all alleles
if limit is not None and line_counter > limit:
break
return
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
src_key = 'catalog'
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
fname = '/'.join((self.rawdir, self.files[src_key]['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = self.globaltt['stem cell']
mouse_taxon = self.globaltt['Mus musculus']
geno = Genotype(graph)
with open(fname, 'r', encoding="utf8") as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='\"')
# First line is header not date/version info. This changed recently,
# apparently as of Sep 2019. Also, 3rd line is no longer blank.
row = [x.strip() for x in next(reader)] # messy messy
col = self.files['catalog']['columns']
strain_missing_allele = [] # to count the ones w/insufficent info
if not self.check_fileheader(col, row):
pass
for row in reader:
strain_id = row[col.index('STRAIN/STOCK_ID')].strip()
strain_label = row[col.index('STRAIN/STOCK_DESIGNATION')]
# strain_type_symbol = row[col.index('STRAIN_TYPE')]
strain_state = row[col.index('STATE')]
mgi_allele_id = row[col.index(
'MGI_ALLELE_ACCESSION_ID')].strip()
mgi_allele_symbol = row[col.index('ALLELE_SYMBOL')]
# mgi_allele_name = row[col.index('ALLELE_NAME')]
# mutation_type = row[col.index('MUTATION_TYPE')]
# chrom = row[col.index('CHROMOSOME')]
mgi_gene_id = row[col.index('MGI_GENE_ACCESSION_ID')].strip()
mgi_gene_symbol = row[col.index('GENE_SYMBOL')].strip()
mgi_gene_name = row[col.index('GENE_NAME')]
# sds_url = row[col.index('SDS_URL')]
# accepted_date = row[col.index('ACCEPTED_DATE')]
mpt_ids = row[col.index('MPT_IDS')].strip()
pubmed_nums = row[col.index('PUBMED_IDS')].strip()
research_areas = row[col.index('RESEARCH_AREAS')].strip()
if self.test_mode and (strain_id not in self.test_ids) \
or mgi_gene_name == 'withdrawn':
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {
'variants': set(),
'genes': set()
}
# flag bad ones
if mgi_allele_id[:4] != 'MGI:' and mgi_allele_id != '':
LOG.error("Erroneous MGI allele id: %s", mgi_allele_id)
if mgi_allele_id[:3] == 'MG:':
mgi_allele_id = 'MGI:' + mgi_allele_id[3:]
else:
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the sequence alteration types
# var_type = self.localtt[mutation_type]
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id, mgi_allele_id)
# scrub out any spaces, fix known issues
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id == 'NULL':
mgi_gene_id = ''
elif mgi_gene_id[:7] == 'GeneID:':
mgi_gene_id = 'NCBIGene:' + mgi_gene_id[7:]
if mgi_gene_id != '':
try:
[curie, localid] = mgi_gene_id.split(':')
except ValueError as verror:
LOG.warning(
"Problem parsing mgi_gene_id %s from file %s: %s",
mgi_gene_id, fname, verror)
if curie not in ['MGI', 'NCBIGene']:
LOG.info("MGI Gene id not recognized: %s", mgi_gene_id)
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors - too many. report summary at the end
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol != '' and mgi_gene_id == '':
# LOG.error(
# "Gene label with no MGI identifier for strain %s: %s",
# strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol)
# make a temp id for genes that aren't identified ... err wow.
# tmp_gene_id = '_' + mgi_gene_symbol
# self.id_label_hash[tmp_gene_id.strip()] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mpt_ids are a comma delimited list
# labels with MP terms following in brackets
phenotype_ids = []
if mpt_ids != '':
for lb_mp in mpt_ids.split(r','):
lb_mp = lb_mp.strip()
if lb_mp[-1:] == ']' and lb_mp[-12:-8] == '[MP:':
phenotype_ids.append(lb_mp[-11:-2])
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums != '':
for pm_num in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:' + pm_num.strip()
pubmed_ids.append(pmid)
ref = Reference(graph, pmid,
self.globaltt['journal article'])
ref.addRefToGraph()
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
model.addClassToGraph(mouse_taxon, None)
if research_areas == '':
research_areas = None
else:
research_areas = 'Research Areas: ' + research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
model.addIndividualToGraph( # an inst of mouse??
strain_id, strain_label, strain_type, research_areas)
model.makeLeader(strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in some ontology
model.addClassToGraph(pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(graph, self.name, mgi_allele_id, pid,
self.globaltt['has phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph()
else:
# too chatty here. report aggregate
# LOG.info("Phenotypes and no allele for %s", strain_id)
strain_missing_allele.append(strain_id)
if not self.test_mode and (limit is not None
and reader.line_num > limit):
break
# report misses
if strain_missing_allele:
LOG.info("Phenotypes and no allele for %i strains",
len(strain_missing_allele))
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if variants:
for var in variants:
vl_id = var.strip()
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(vl_id, vl_symbol,
self.globaltt['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_:' + re.sub(r':', '', gene) + '-VL'
vl_symbol = self.id_label_hash[gene] + '<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(vl_id, vl_symbol,
self.globaltt['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = re.sub(r'^_', '', vl) + 'U'
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(vslc_id, vl, None,
self.globaltt['indeterminate'],
self.globaltt['has_variant_part'],
None)
model.addIndividualToGraph(
vslc_id, vslc_label,
self.globaltt['variant single locus complement'])
if vslc_list:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r'_|:', '', gvc_id)
gvc_id = '_:' + gvc_id
gvc_label = '; '.join(self.id_label_hash[v]
for v in vslc_list)
model.addIndividualToGraph(
gvc_id, gvc_label,
self.globaltt['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = re.sub(
r':', '', '-'.join(
(self.globaltt['unspecified_genomic_background'],
s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
bkgd_id = '_:' + bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified (' + s + ')',
self.globaltt['unspecified_genomic_background'],
"A placeholder for the unspecified genetic background for "
+ s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
self.globaltt['unspecified_genomic_background'])
geno.addParts(gvc_id, genotype_id,
self.globaltt['has_variant_part'])
geno.addGenotype(genotype_id, genotype_label)
graph.addTriple(s, self.globaltt['has_genotype'],
genotype_id)
else:
# LOG.debug(
# "Strain %s is not making a proper genotype.", s)
pass
LOG.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
LOG.error('%i symbols given are missing their gene identifiers',
len(genes_with_no_ids))
return
def _get_process_allelic_variants(self, entry, g):
model = Model(g)
reference = Reference(g)
geno = Genotype(g)
if entry is not None:
# to hold the entry-specific publication mentions
# for the allelic variants
publist = {}
entry_num = entry['mimNumber']
# process the ref list just to get the pmids
ref_to_pmid = self._get_pubs(entry, g)
if 'allelicVariantList' in entry:
allelicVariantList = entry['allelicVariantList']
for al in allelicVariantList:
al_num = al['allelicVariant']['number']
al_id = 'OMIM:'+str(entry_num)+'.'+str(al_num).zfill(4)
al_label = None
al_description = None
if al['allelicVariant']['status'] == 'live':
publist[al_id] = set()
if 'mutations' in al['allelicVariant']:
al_label = al['allelicVariant']['mutations']
if 'text' in al['allelicVariant']:
al_description = al['allelicVariant']['text']
m = re.findall(r'\{(\d+)\:', al_description)
publist[al_id] = set(m)
geno.addAllele(
al_id, al_label, geno.genoparts['variant_locus'],
al_description)
geno.addAlleleOfGene(
al_id, 'OMIM:'+str(entry_num),
geno.object_properties[
'is_sequence_variant_instance_of'])
for r in publist[al_id]:
pmid = ref_to_pmid[int(r)]
g.addTriple(
pmid, model.object_properties['is_about'],
al_id)
# look up the pubmed id in the list of references
if 'dbSnps' in al['allelicVariant']:
dbsnp_ids = \
re.split(r',', al['allelicVariant']['dbSnps'])
for dnum in dbsnp_ids:
did = 'dbSNP:'+dnum.strip()
model.addIndividualToGraph(did, None)
model.addSameIndividual(al_id, did)
if 'clinvarAccessions' in al['allelicVariant']:
# clinvarAccessions triple semicolon delimited
# each >1 like RCV000020059;;;
rcv_ids = \
re.split(
r';;;',
al['allelicVariant']['clinvarAccessions'])
rcv_ids = [
(re.match(r'(RCV\d+);*', r)).group(1)
for r in rcv_ids]
for rnum in rcv_ids:
rid = 'ClinVar:'+rnum
model.addXref(al_id, rid)
reference.addPage(
al_id, "http://omim.org/entry/" +
str(entry_num)+"#" + str(al_num).zfill(4))
elif re.search(
r'moved', al['allelicVariant']['status']):
# for both 'moved' and 'removed'
moved_ids = None
if 'movedTo' in al['allelicVariant']:
moved_id = 'OMIM:'+al['allelicVariant']['movedTo']
moved_ids = [moved_id]
model.addDeprecatedIndividual(al_id, moved_ids)
else:
logger.error('Uncaught alleleic variant status %s',
al['allelicVariant']['status'])
# end loop allelicVariantList
return
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
src_key = 'catalog'
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
fname = '/'.join((self.rawdir, self.files[src_key]['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = self.globaltt['stem cell']
mouse_taxon = self.globaltt['Mus musculus']
geno = Genotype(graph)
with open(fname, 'r', encoding="utf8") as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='\"')
# This MMRRC catalog data file was generated on YYYY-MM-DD
# insert or check date w/dataset
line = next(reader)
# gen_date = line[-10:]
line = next(reader)
col = self.files['catalog']['columns']
if col != line:
LOG.error(
'%s\nExpected Headers:\t%s\nRecived Headers:\t%s\n',
src_key, col, line)
LOG.info(set(col) - set(line))
line = next(reader)
if line != []:
LOG.warning('Expected third line to be blank. got "%s" instead', line)
for row in reader:
strain_id = row[col.index('STRAIN/STOCK_ID')].strip()
strain_label = row[col.index('STRAIN/STOCK_DESIGNATION')]
# strain_type_symbol = row[col.index('STRAIN_TYPE')]
strain_state = row[col.index('STATE')]
mgi_allele_id = row[col.index('MGI_ALLELE_ACCESSION_ID')].strip()
mgi_allele_symbol = row[col.index('ALLELE_SYMBOL')]
# mgi_allele_name = row[col.index('ALLELE_NAME')]
# mutation_type = row[col.index('MUTATION_TYPE')]
# chrom = row[col.index('CHROMOSOME')]
mgi_gene_id = row[col.index('MGI_GENE_ACCESSION_ID')].strip()
mgi_gene_symbol = row[col.index('GENE_SYMBOL')].strip()
mgi_gene_name = row[col.index('GENE_NAME')]
# sds_url = row[col.index('SDS_URL')]
# accepted_date = row[col.index('ACCEPTED_DATE')]
mpt_ids = row[col.index('MPT_IDS')].strip()
pubmed_nums = row[col.index('PUBMED_IDS')].strip()
research_areas = row[col.index('RESEARCH_AREAS')].strip()
if self.test_mode and (strain_id not in self.test_ids) \
or mgi_gene_name == 'withdrawn':
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {
'variants': set(), 'genes': set()}
# flag bad ones
if mgi_allele_id[:4] != 'MGI:' and mgi_allele_id != '':
LOG.error("Erroneous MGI allele id: %s", mgi_allele_id)
if mgi_allele_id[:3] == 'MG:':
mgi_allele_id = 'MGI:' + mgi_allele_id[3:]
else:
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the sequence alteration types
# var_type = self.localtt[mutation_type]
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id, mgi_allele_id)
# scrub out any spaces, fix known issues
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id == 'NULL':
mgi_gene_id = ''
elif mgi_gene_id[:7] == 'GeneID:':
mgi_gene_id = 'NCBIGene:' + mgi_gene_id[7:]
if mgi_gene_id != '':
[curie, localid] = mgi_gene_id.split(':')
if curie not in ['MGI', 'NCBIGene']:
LOG.info("MGI Gene id not recognized: %s", mgi_gene_id)
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors - too many. report summary at the end
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol != '' and mgi_gene_id == '':
# LOG.error(
# "Gene label with no MGI identifier for strain %s: %s",
# strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol)
# make a temp id for genes that aren't identified ... err wow.
# tmp_gene_id = '_' + mgi_gene_symbol
# self.id_label_hash[tmp_gene_id.strip()] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mpt_ids are a comma delimited list
# labels with MP terms following in brackets
phenotype_ids = []
if mpt_ids != '':
for lb_mp in mpt_ids.split(r','):
lb_mp = lb_mp.strip()
if lb_mp[-1:] == ']' and lb_mp[-12:-8] == '[MP:':
phenotype_ids.append(lb_mp[-11:-2])
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums != '':
for pm_num in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:' + pm_num.strip()
pubmed_ids.append(pmid)
ref = Reference(graph, pmid, self.globaltt['journal article'])
ref.addRefToGraph()
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
model.addClassToGraph(mouse_taxon, None)
if research_areas == '':
research_areas = None
else:
research_areas = 'Research Areas: ' + research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
model.addIndividualToGraph( # an inst of mouse??
strain_id, strain_label, strain_type, research_areas)
model.makeLeader(strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in some ontology
model.addClassToGraph(pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(
graph, self.name, mgi_allele_id, pid,
self.globaltt['has phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph()
else:
LOG.info("Phenotypes and no allele for %s", strain_id)
if not self.test_mode and (
limit is not None and reader.line_num > limit):
break
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if len(variants) > 0:
for var in variants:
vl_id = var.strip()
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(
vl_id, vl_symbol, self.globaltt['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_:' + re.sub(r':', '', gene) + '-VL'
vl_symbol = self.id_label_hash[gene]+'<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(
vl_id, vl_symbol, self.globaltt['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = re.sub(r'^_', '', vl)+'U'
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(
vslc_id, vl, None, self.globaltt['indeterminate'],
self.globaltt['has_variant_part'], None)
model.addIndividualToGraph(
vslc_id, vslc_label,
self.globaltt['variant single locus complement'])
if len(vslc_list) > 0:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r'_|:', '', gvc_id)
gvc_id = '_:'+gvc_id
gvc_label = '; '.join(self.id_label_hash[v] for v in vslc_list)
model.addIndividualToGraph(
gvc_id, gvc_label,
self.globaltt['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = re.sub(
r':', '', '-'.join((
self.globaltt['unspecified_genomic_background'], s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
bkgd_id = '_:' + bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified (' + s + ')',
self.globaltt['unspecified_genomic_background'],
"A placeholder for the unspecified genetic background for " + s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
self.globaltt['unspecified_genomic_background'])
geno.addParts(
gvc_id, genotype_id, self.globaltt['has_variant_part'])
geno.addGenotype(genotype_id, genotype_label)
graph.addTriple(
s, self.globaltt['has_genotype'], genotype_id)
else:
# LOG.debug(
# "Strain %s is not making a proper genotype.", s)
pass
LOG.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
LOG.error(
'%i symbols given are missing their gene identifiers',
len(genes_with_no_ids))
return
class Decipher(Source):
"""
The Decipher group curates and assembles the Development Disorder Genotype
Phenotype Database (DDG2P) which is a curated list of genes reported to be
associated with developmental disorders, compiled by clinicians as part of
the DDD study to facilitate clinical feedback of likely causal variants.
Beware that the redistribution of this data is a bit unclear from the
[license](https://decipher.sanger.ac.uk/legal).
If you intend to distribute this data, be sure to have the appropriate
licenses in place.
"""
files = {
'annot': {
'file': 'ddg2p.zip',
'url': 'https://decipher.sanger.ac.uk/files/ddd/ddg2p.zip'}
}
def __init__(self):
Source.__init__(self, 'decipher')
self.load_bindings()
self.dataset = Dataset(
'decipher', 'Development Disorder Genotype – Phenotype Database',
'https://decipher.sanger.ac.uk/', None,
'https://decipher.sanger.ac.uk/legal')
if 'test_ids' not in config.get_config() \
or 'disease' not in config.get_config()['test_ids']:
logger.warning("not configured with disease test ids.")
self.test_ids = []
else:
self.test_ids = config.get_config()['test_ids']['disease']
self.gu = GraphUtils(curie_map.get())
self.g = self.graph
self.geno = Genotype(self.g)
return
def fetch(self, is_dl_forced=False):
self.get_files(is_dl_forced)
# since there's a dependency on HGNC files; fetch those too
hgnc = HGNC()
hgnc.fetch(is_dl_forced)
return
def parse(self, limit=None):
if limit is not None:
logger.info("Only parsing first %s rows", limit)
logger.info("Parsing files...")
if self.testOnly:
self.testMode = True
self.g = self.testgraph
else:
self.g = self.graph
self.geno = Genotype(self.g)
# rare disease-phenotype associations
self._process_ddg2p_annotations(limit)
logger.info("Finished parsing.")
return
def _process_ddg2p_annotations(self, limit):
"""
The ddg2p annotations associate a gene symbol to an omim disease,
along with some HPO ids and pubs. The gene symbols come from gencode,
which in turn come from HGNC official gene symbols. Therefore,
we use the HGNC source class to get the id/symbol mapping for
use in our annotations here.
According to http://www.gencodegenes.org/faq.html,
"Gene names are usually HGNC or MGI-approved gene symbols mapped
to the GENCODE genes by the Ensembl xref pipeline. Sometimes,
when there is no official gene symbol, the Havana clone-based
name is used."
The kind of variation that is linked to a disease is indicated
(LOF, GOF, CNV, etc) in the source data.
Here, we create an anonymous variant of the specified gene of
the indicated type (mapped to the sequence ontology (SO)).
:param limit:
:return:
"""
line_counter = 0
if self.g is not None:
g = self.g
else:
g = self.graph
gu = GraphUtils(curie_map.get())
# in order for this to work, we need to map the HGNC id-symbol;
hgnc = HGNC()
hgnc_symbol_id_map = hgnc.get_symbol_id_map()
myzip = ZipFile(
'/'.join((self.rawdir, self.files['annot']['file'])), 'r')
# use the ddg2p.txt file
fname = 'ddg2p.txt'
unmapped_omim_counter = 0
unmapped_gene_count = 0
with myzip.open(fname, 'r') as f:
f = io.TextIOWrapper(f)
reader = csv.reader(f, delimiter='\t', quotechar='\"')
# score_means_by_measure = {}
# strain_scores_by_measure = {} # TODO theseare unused
for row in reader:
line_counter += 1
if re.match(r'#', row[0]): # skip comments
continue
(gencode_gene_name, mode, category, consequence, disease, omim,
ddg2p_id, pubmed_ids, hpo_codes) = row
hgnc_id = hgnc_symbol_id_map.get(gencode_gene_name.strip())
if hgnc_id is None:
logger.error(
"Couldn't map the gene symbol %s to HGNC.",
gencode_gene_name)
unmapped_gene_count += 1
continue
# add the gene
gu.addClassToGraph(g, hgnc_id, gencode_gene_name)
# TODO make VSLC with the variation
# to associate with the disorder
# TODO use the Inheritance and Mutation consequence
# to classify the VSLCs
allele_id = self.make_allele_by_consequence(
consequence, hgnc_id, gencode_gene_name)
if omim.strip() != '':
omim_id = 'OMIM:'+str(omim.strip())
# assume this is declared elsewhere in ontology
gu.addClassToGraph(g, omim_id, None)
if category.strip() == 'Confirmed DD gene':
rel = gu.object_properties['has_phenotype']
elif category.strip() == 'Probable DD gene':
rel = gu.object_properties['has_phenotype']
elif category.strip() == 'Possible DD gene':
rel = gu.object_properties['contributes_to']
elif category.strip() == 'Not DD gene':
# TODO negative annotation
continue
assoc = G2PAssoc(self.name, allele_id, omim_id)
# TODO 'rel' is assigned to but never used
for p in re.split(r';', pubmed_ids):
p = p.strip()
if p != '':
pmid = 'PMID:'+str(p)
r = Reference(
pmid, Reference.ref_types['journal_article'])
r.addRefToGraph(g)
assoc.add_source(pmid)
assoc.add_association_to_graph(g)
else:
# these are unmapped to a disease id.
# note that some match OMIM disease labels
# but the identifiers are just not included.
# TODO consider mapping to OMIM or DOIDs in other ways
logger.warning(
"No omim id on line %d\n%s", line_counter, str(row))
unmapped_omim_counter += 1
# TODO hpo phenotypes
# since the DDG2P file is not documented,
# I don't know what the HPO annotations are actually about
# are they about the gene? the omim disease? something else?
# So, we wont create associations until this is clarified
if not self.testMode and limit is not None \
and line_counter > limit:
break
myzip.close()
logger.warning(
"gene-disorder associations with no omim id: %d",
unmapped_omim_counter)
logger.warning("unmapped gene count: %d", unmapped_gene_count)
gu.loadProperties(g, G2PAssoc.object_properties, gu.OBJPROP)
gu.loadProperties(g, G2PAssoc.datatype_properties, gu.DATAPROP)
gu.loadProperties(g, G2PAssoc.annotation_properties, gu.ANNOTPROP)
return
def make_allele_by_consequence(self, consequence, gene_id, gene_symbol):
"""
Given a "consequence" label that describes a variation type,
create an anonymous variant of the specified gene as an instance of
that consequence type.
:param consequence:
:param gene_id:
:param gene_symbol:
:return: allele_id
"""
allele_id = None
# Loss of function : Nonsense, frame-shifting indel,
# essential splice site mutation, whole gene deletion or any other
# mutation where functional analysis demonstrates clear reduction
# or loss of function
# All missense/in frame : Where all the mutations described in the data
# source are either missense or in frame deletions and there is no
# evidence favoring either loss-of-function, activating or
# dominant negative effect
# Dominant negative : Mutation within one allele of a gene that creates
# a significantly greater deleterious effect on gene product
# function than a monoallelic loss of function mutation
# Activating : Mutation, usually missense that results in
# a constitutive functional activation of the gene product
# Increased gene dosage : Copy number variation that increases
# the functional dosage of the gene
# Cis-regulatory or promotor mutation : Mutation in cis-regulatory
# elements that lies outwith the known transcription unit and
# promotor of the controlled gene
# Uncertain : Where the exact nature of the mutation is unclear or
# not recorded
so_type = { # type of variant
'Loss of function': 'SO:0002054', # loss of function
'All missense/in frame': 'SO:0001583', # missense
'Dominant negative': 'SO:0002052', # dominant negative
'Activating': 'SO:0002053', # gain of function
'Increased gene dosage': 'SO:0001742', # copy number gain
# regulatory region
'Cis-regulatory or promotor mutation': 'SO:0001566',
'Uncertain': 'SO:0001060', # generic sequence
'5 or 3UTR mutation': 'SO:0001622', # UTR
}
type_id = so_type.get(consequence)
if type_id is None:
logger.warning("Consequence type unmapped: %s", str(consequence))
type_id = 'SO:0001060' # sequence variant
# make the allele
allele_id = ''.join((gene_id, type_id))
allele_id = re.sub(r':', '', allele_id)
allele_id = '_'+allele_id # make this a BNode
if self.nobnodes:
allele_id = ':'+allele_id
allele_label = ' '.join((consequence, 'allele in', gene_symbol))
self.gu.addIndividualToGraph(self.g, allele_id, allele_label, type_id)
self.geno.addAlleleOfGene(allele_id, gene_id)
return allele_id
def process_allele_phenotype(self, limit=None):
"""
This file compactly lists variant to phenotype associations,
such that in a single row, there may be >1 variant listed
per phenotype and paper. This indicates that each variant is
individually assocated with the given phenotype,
as listed in 1+ papers.
(Not that the combination of variants is producing the phenotype.)
:param limit:
:return:
"""
src_key = 'allele_pheno'
raw = '/'.join((self.rawdir, self.files[src_key]['file']))
col = self.files[src_key]['columns']
graph = self.graph
model = Model(self.graph)
LOG.info("Processing Allele phenotype associations")
geno = Genotype(graph)
with open(raw, 'r') as csvfile:
reader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
row = next(reader)
if row[0] != '!gaf-version: 2.0':
LOG.error('Not a vlaid gaf v2.0 formatted file: %s', raw)
# raise
for row in reader:
if row[0][0] == '!':
continue
# db = row[col.index('DB')]
gene_num = row[col.index('DB Object ID')]
# gene_symbol = row[col.index('DB Object Symbol')]
is_not = row[col.index('Qualifier')]
phenotype_id = row[col.index('GO ID')]
ref = row[col.index('DB:Reference (|DB:Reference)')].strip()
eco_symbol = row[col.index('Evidence Code')]
with_or_from = row[col.index('With (or) From')]
# aspect = row[col.index('Aspect')]
# gene_name = row[col.index('DB Object Name')]
# gene_synonym = row[col.index('DB Object Synonym (|Synonym)')]
# gene_class = row[col.index('DB Object Type')]
# taxon = row[col.index('Taxon(|taxon)')]
# date = row[col.index('Date')]
# assigned_by = row[col.index('Assigned By')]
# blank = row[col.index('Annotation Extension')]
# blank2 = row[col.index('Gene Product Form ID')]
# TODO add NOT phenotypes
if is_not == 'NOT':
continue
eco_symbol = eco_symbol.strip()
eco_curie = None
if eco_symbol.strip() != '' and eco_symbol in self.gaf_eco:
eco_curie = self.gaf_eco[eco_symbol]
else:
LOG.warning(
'Evidence code %s is not found in the (gaf) gaf_eco',
eco_symbol)
# according to the GOA spec, persons are not allowed to be
# in the reference column, therefore they the variant and
# persons are swapped between the reference and with column.
# we unswitch them here.
temp_var = temp_ref = None
if re.search(r'WBVar|WBRNAi', ref):
temp_var = ref
# move the paper from the with column into the ref
if re.search(r'WBPerson', with_or_from):
temp_ref = with_or_from
if temp_var is not None or temp_ref is not None:
with_or_from = temp_var
ref = temp_ref
allele_list = re.split(r'\|', with_or_from)
if len(allele_list) == 0:
LOG.error(
"Missing alleles from phenotype assoc at line %d",
reader.line_num)
continue
else:
for allele in allele_list:
allele_num = re.sub(r'WB:', '', allele.strip())
allele_id = 'WormBase:' + allele_num
gene_id = 'WormBase:' + gene_num
if re.search(r'WBRNAi', allele_id):
# @kshefchek - removing this blank node
# in favor of simpler modeling
# make the WormBase:WBRNAi* id
# a self.globaltt['reagent_targeted_gene'], and attach
# phenotype to this ID
# Previous model - make a bnode reagent-targeted gene,
# & annotate that instead of the RNAi item directly
# rnai_num = re.sub(r'WormBase:', '', allele_id)
# rnai_id = allele_id
# rtg_id = self.make_reagent_targeted_gene_id(
# gene_num, rnai_num)
# geno.addReagentTargetedGene(
# rnai_id, 'WormBase:' + gene_num, rtg_id)
# allele_id = rtg_id
# Could type the IRI as both the reagant and reagant
# targeted gene but not sure if this needed
# geno.addGeneTargetingReagent(
# allele_id, None, self.globaltt['RNAi_reagent'], gene_id)
model.addIndividualToGraph(
allele_id, None,
self.globaltt['reagent_targeted_gene'])
self.graph.addTriple(
allele_id,
self.globaltt['is_expression_variant_of'],
gene_id)
elif re.search(r'WBVar', allele_id):
# this may become deprecated by using wormmine
# make the allele to gene relationship
# the WBVars are really sequence alterations
# the public name will come from elsewhere
# @kshefchek - removing this blank node
# in favor of simpler modeling, treat variant
# like an allele
# vl_id = '_:'+'-'.join((gene_num, allele_num))
# geno.addSequenceAlterationToVariantLocus(
# allele_id, vl_id)
# geno.addAlleleOfGene(vl_id, gene_id)
geno.addSequenceAlteration(allele_id, None)
geno.addAlleleOfGene(allele_id, gene_id)
else:
LOG.warning(
"Some kind of allele I don't recognize: %s",
allele_num)
continue
assoc = G2PAssoc(graph, self.name, allele_id,
phenotype_id)
if eco_curie is not None:
assoc.add_evidence(eco_curie)
if ref is not None and ref != '':
ref = re.sub(r'(WB:|WB_REF:)', 'WormBase:', ref)
reference = Reference(graph, ref)
if re.search(r'Person', ref):
reference.setType(self.globaltt['person'])
assoc.add_evidence(self.globaltt[
'inference from background scientific knowledge']
)
reference.addRefToGraph()
assoc.add_source(ref)
assoc.add_association_to_graph()
# finish looping through all alleles
if limit is not None and reader.line_num > limit:
break
def process_allele_phenotype(self, limit=None):
"""
This file compactly lists variant to phenotype associations,
such that in a single row, there may be >1 variant listed
per phenotype and paper. This indicates that each variant is
individually assocated with the given phenotype,
as listed in 1+ papers.
(Not that the combination of variants is producing the phenotype.)
:param limit:
:return:
"""
raw = '/'.join((self.rawdir, self.files['allele_pheno']['file']))
if self.testMode:
g = self.testgraph
else:
g = self.graph
logger.info("Processing Allele phenotype associations")
line_counter = 0
geno = Genotype(g)
with open(raw, 'r') as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
if re.match(r'!', ''.join(row)): # header
continue
line_counter += 1
(db, gene_num, gene_symbol, is_not, phenotype_id, ref,
eco_symbol, with_or_from, aspect, gene_name, gene_synonym,
gene_class, taxon, date, assigned_by, blank, blank2) = row
if self.testMode and gene_num not in self.test_ids['gene']:
continue
# TODO add NOT phenotypes
if is_not == 'NOT':
continue
eco_id = None
if eco_symbol == 'IMP':
eco_id = 'ECO:0000015'
elif eco_symbol.strip() != '':
logger.warning("Encountered an ECO code we don't have: %s",
eco_symbol)
# according to the GOA spec, persons are not allowed to be
# in the reference column, therefore they the variant and
# persons are swapped between the reference and with column.
# we unswitch them here.
temp_var = temp_ref = None
if re.search(r'WBVar|WBRNAi', ref):
temp_var = ref
# move the paper from the with column into the ref
if re.search(r'WBPerson', with_or_from):
temp_ref = with_or_from
if temp_var is not None or temp_ref is not None:
with_or_from = temp_var
ref = temp_ref
allele_list = re.split(r'\|', with_or_from)
if len(allele_list) == 0:
logger.error(
"Missing alleles from phenotype assoc at line %d",
line_counter)
continue
else:
for a in allele_list:
allele_num = re.sub(r'WB:', '', a.strip())
allele_id = 'WormBase:' + allele_num
gene_id = 'WormBase:' + gene_num
if re.search(r'WBRNAi', allele_id):
# make the reagent-targeted gene,
# & annotate that instead of the RNAi item directly
rnai_num = re.sub(r'WormBase:', '', allele_id)
rnai_id = allele_id
rtg_id = self.make_reagent_targeted_gene_id(
gene_num, rnai_num)
geno.addReagentTargetedGene(
rnai_id, 'WormBase:' + gene_num, rtg_id)
geno.addGeneTargetingReagent(
rnai_id, None, geno.genoparts['RNAi_reagent'],
gene_id)
allele_id = rtg_id
elif re.search(r'WBVar', allele_id):
# this may become deprecated by using wormmine
# make the allele to gene relationship
# the WBVars are really sequence alterations
# the public name will come from elsewhere
geno.addSequenceAlteration(allele_id, None)
vl_id = '_:' + '-'.join((gene_num, allele_num))
geno.addSequenceAlterationToVariantLocus(
allele_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
logger.warning(
"Some kind of allele I don't recognize: %s",
allele_num)
continue
assoc = G2PAssoc(g, self.name, allele_id, phenotype_id)
if eco_id is not None:
assoc.add_evidence(eco_id)
if ref is not None and ref != '':
ref = re.sub(r'(WB:|WB_REF:)', 'WormBase:', ref)
reference = Reference(g, ref)
if re.search(r'Person', ref):
reference.setType(
reference.ref_types['person'])
# also add
# inferred from background scientific knowledge
assoc.add_evidence('ECO:0000001')
reference.addRefToGraph()
assoc.add_source(ref)
assoc.add_association_to_graph()
# finish looping through all alleles
if not self.testMode \
and limit is not None and line_counter > limit:
break
return
class Decipher(Source):
"""
Deprecated - please see the EBIGene2Phen class, which parses the same
file but fetches it from EBI which has clearer terms for redistribution,
while Decipher has restrictive terms due to containing patient data in
password protected datasets.
The Decipher group curates and assembles the Development Disorder Genotype
Phenotype Database (DDG2P) which is a curated list of genes reported to be
associated with developmental disorders, compiled by clinicians as part of
the DDD study to facilitate clinical feedback of likely causal variants.
Beware that the redistribution of this data is a bit unclear from the
[license](https://decipher.sanger.ac.uk/legal).
If you intend to distribute this data, be sure to have the appropriate
licenses in place.
"""
files = {
'annot': {
'file': 'ddg2p.zip',
'url': 'https://decipher.sanger.ac.uk/files/ddd/ddg2p.zip',
'headers': []
}
}
def __init__(self, graph_type, are_bnodes_skolemized):
super().__init__(
graph_type,
are_bnodes_skolemized,
'decipher',
ingest_title='Development Disorder Genotype Phenotype Database',
ingest_url='https://decipher.sanger.ac.uk/',
license_url='https://decipher.sanger.ac.uk/legal',
data_rights='https://decipher.sanger.ac.uk/datasharing',
# file_handle=None
)
if 'disease' not in self.all_test_ids:
LOG.warning("not configured with disease test ids.")
self.test_ids = []
else:
self.test_ids = self.all_test_ids['disease']
self.graph = self.graph
self.geno = Genotype(self.graph)
self.model = Model(self.graph)
self.graph_type = graph_type
self.are_bnodes_skolemized = are_bnodes_skolemized
return
def fetch(self, is_dl_forced=False):
self.get_files(is_dl_forced)
# since there's a dependency on HGNC files; fetch those too
hgnc = HGNC(self.graph_type, self.are_bnodes_skolemized)
hgnc.fetch(is_dl_forced)
return
def parse(self, limit=None):
if limit is not None:
LOG.info("Only parsing first %s rows", limit)
LOG.info("Parsing files...")
if self.test_only:
self.test_mode = True
self.graph = self.testgraph
else:
self.graph = self.graph
self.geno = Genotype(self.graph)
# rare disease-phenotype associations
self._process_ddg2p_annotations(limit)
LOG.info("Finished parsing.")
return
def _process_ddg2p_annotations(self, limit):
"""
The ddg2p annotations associate a gene symbol to an omim disease,
along with some HPO ids and pubs. The gene symbols come from gencode,
which in turn come from HGNC official gene symbols. Therefore,
we use the HGNC source class to get the id/symbol mapping for
use in our annotations here.
According to http://www.gencodegenes.org/faq.html,
"Gene names are usually HGNC or MGI-approved gene symbols mapped
to the GENCODE genes by the Ensembl xref pipeline. Sometimes,
when there is no official gene symbol, the Havana clone-based
name is used."
The kind of variation that is linked to a disease is indicated
(LOF, GOF, CNV, etc) in the source data.
Here, we create an anonymous variant of the specified gene of
the indicated type (mapped to the sequence ontology (SO)).
:param limit:
:return:
"""
line_counter = 0
if self.graph is not None:
graph = self.graph
else:
graph = self.graph
# in order for this to work, we need to map the HGNC id-symbol;
hgnc = HGNC(self.graph_type, self.are_bnodes_skolemized)
hgnc_symbol_id_map = hgnc.get_symbol_id_map()
myzip = ZipFile('/'.join((self.rawdir, self.files['annot']['file'])),
'r')
# use the ddg2p.txt file
fname = 'ddg2p.txt'
unmapped_omim_counter = 0
unmapped_gene_count = 0
with myzip.open(fname, 'r') as f:
f = io.TextIOWrapper(f)
reader = csv.reader(f, delimiter='\t', quotechar='\"')
# score_means_by_measure = {}
# strain_scores_by_measure = {} # TODO theseare unused
for row in reader:
line_counter += 1
if re.match(r'#', row[0]): # skip comments
continue
(gencode_gene_name, mode, category, consequence, disease, omim,
ddg2p_id, pubmed_ids, hpo_codes) = row
hgnc_id = hgnc_symbol_id_map.get(gencode_gene_name.strip())
if hgnc_id is None:
LOG.error("Couldn't map the gene symbol %s to HGNC.",
gencode_gene_name)
unmapped_gene_count += 1
continue
# add the gene
self.model.addClassToGraph(hgnc_id, gencode_gene_name)
# TODO make VSLC with the variation
# to associate with the disorder
# TODO use the Inheritance and Mutation consequence
# to classify the VSLCs
allele_id = self.make_allele_by_consequence(
consequence, hgnc_id, gencode_gene_name)
if omim.strip() != '':
omim_id = 'OMIM:' + str(omim.strip())
# assume this is declared elsewhere in ontology
self.model.addClassToGraph(omim_id, None)
# ??? rel is never used
# if category.strip() == 'Confirmed DD gene':
# rel = self.self.globaltt['has phenotype']
# elif category.strip() == 'Probable DD gene':
# rel = self.self.globaltt['has phenotype']
# elif category.strip() == 'Possible DD gene':
# rel = self.self.globaltt['contributes to']
# elif category.strip() == 'Not DD gene':
# # TODO negative annotation
# continue
assoc = G2PAssoc(graph, self.name, allele_id, omim_id)
# TODO 'rel' is assigned to but never used
for p in re.split(r';', pubmed_ids):
p = p.strip()
if p != '':
pmid = 'PMID:' + str(p)
r = Reference(graph, pmid,
self.globaltt['journal article'])
r.addRefToGraph()
assoc.add_source(pmid)
assoc.add_association_to_graph()
else:
# these are unmapped to a disease id.
# note that some match OMIM disease labels
# but the identifiers are just not included.
# TODO consider mapping to OMIM or DOIDs in other ways
LOG.warning("No omim id on line %d\n%s", line_counter,
str(row))
unmapped_omim_counter += 1
# TODO hpo phenotypes
# since the DDG2P file is not documented,
# I don't know what the HPO annotations are actually about
# are they about the gene? the omim disease? something else?
# So, we wont create associations until this is clarified
if not self.test_mode and limit is not None and line_counter > limit:
break
myzip.close()
LOG.warning("gene-disorder associations with no omim id: %d",
unmapped_omim_counter)
LOG.warning("unmapped gene count: %d", unmapped_gene_count)
return
def make_allele_by_consequence(self, consequence, gene_id, gene_symbol):
"""
Given a "consequence" label that describes a variation type,
create an anonymous variant of the specified gene as an instance of
that consequence type.
:param consequence:
:param gene_id:
:param gene_symbol:
:return: allele_id
"""
allele_id = None
# Loss of function : Nonsense, frame-shifting indel,
# essential splice site mutation, whole gene deletion or any other
# mutation where functional analysis demonstrates clear reduction
# or loss of function
# All missense/in frame : Where all the mutations described in the data
# source are either missense or in frame deletions and there is no
# evidence favoring either loss-of-function, activating or
# dominant negative effect
# Dominant negative : Mutation within one allele of a gene that creates
# a significantly greater deleterious effect on gene product
# function than a monoallelic loss of function mutation
# Activating : Mutation, usually missense that results in
# a constitutive functional activation of the gene product
# Increased gene dosage : Copy number variation that increases
# the functional dosage of the gene
# Cis-regulatory or promotor mutation : Mutation in cis-regulatory
# elements that lies outwith the known transcription unit and
# promotor of the controlled gene
# Uncertain : Where the exact nature of the mutation is unclear or
# not recorded
type_id = self.resolve(consequence, mandatory=False)
if type_id == consequence:
LOG.warning("Consequence type unmapped: %s", str(consequence))
type_id = self.globaltt['sequence_variant']
# make the allele
allele_id = ''.join((gene_id, type_id))
allele_id = re.sub(r':', '', allele_id)
allele_id = '_:' + allele_id # make this a BNode
allele_label = ' '.join((consequence, 'allele in', gene_symbol))
self.model.addIndividualToGraph(allele_id, allele_label, type_id)
self.geno.addAlleleOfGene(allele_id, gene_id)
return allele_id
def _get_variants(self, limit):
"""
Currently loops through the variant_summary file.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
geno = Genotype(g)
f = Feature(g, None, None, None)
# add the taxon and the genome
tax_num = '9606' # HARDCODE
tax_id = 'NCBITaxon:'+tax_num
tax_label = 'Human'
model.addClassToGraph(tax_id, None)
geno.addGenome(tax_id, tax_label) # label gets added elsewhere
# not unzipping the file
logger.info("Processing Variant records")
line_counter = 0
myfile = '/'.join((self.rawdir, self.files['variant_summary']['file']))
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match(r'^#', line):
continue
# AlleleID integer value as stored in the AlleleID field in ClinVar (//Measure/@ID in the XML)
# Type character, the type of variation
# Name character, the preferred name for the variation
# GeneID integer, GeneID in NCBI's Gene database
# GeneSymbol character, comma-separated list of GeneIDs overlapping the variation
# ClinicalSignificance character, comma-separated list of values of clinical significance reported for this variation
# for the mapping between the terms listed here and the integers in the .VCF files, see
# http://www.ncbi.nlm.nih.gov/clinvar/docs/clinsig/
# RS# (dbSNP) integer, rs# in dbSNP
# nsv (dbVar) character, the NSV identifier for the region in dbVar
# RCVaccession character, list of RCV accessions that report this variant
# TestedInGTR character, Y/N for Yes/No if there is a test registered as specific to this variation in the NIH Genetic Testing Registry (GTR)
# PhenotypeIDs character, list of db names and identifiers for phenotype(s) reported for this variant
# Origin character, list of all allelic origins for this variation
# Assembly character, name of the assembly on which locations are based
# Chromosome character, chromosomal location
# Start integer, starting location, in pter->qter orientation
# Stop integer, end location, in pter->qter orientation
# Cytogenetic character, ISCN band
# ReviewStatus character, highest review status for reporting this measure. For the key to the terms,
# and their relationship to the star graphics ClinVar displays on its web pages,
# see http://www.ncbi.nlm.nih.gov/clinvar/docs/variation_report/#interpretation
# HGVS(c.) character, RefSeq cDNA-based HGVS expression
# HGVS(p.) character, RefSeq protein-based HGVS expression
# NumberSubmitters integer, number of submissions with this variant
# LastEvaluated datetime, the latest time any submitter reported clinical significance
# Guidelines character, ACMG only right now, for the reporting of incidental variation in a Gene
# (NOTE: if ACMG, not a specific to the allele but to the Gene)
# OtherIDs character, list of other identifiers or sources of information about this variant
# VariantID integer, the value used to build the URL for the current default report,
# e.g. http://www.ncbi.nlm.nih.gov/clinvar/variation/1756/
#
# a crude check that there's an expected number of cols.
# if not, error out because something changed.
num_cols = len(line.split('\t'))
expected_numcols = 29
if num_cols != expected_numcols:
logger.error(
"Unexpected number of columns in raw file " +
"(%d actual vs %d expected)",
num_cols, expected_numcols)
(allele_num, allele_type, allele_name, gene_num, gene_symbol,
clinical_significance, dbsnp_num, dbvar_num, rcv_nums,
tested_in_gtr, phenotype_ids, origin, assembly, chr, start,
stop, cytogenetic_loc, review_status, hgvs_c, hgvs_p,
number_of_submitters, last_eval, guidelines, other_ids,
variant_num, reference_allele, alternate_allele, categories,
ChromosomeAccession) = line.split('\t')
# ###set filter=None in init if you don't want to have a filter
# if self.filter is not None:
# if ((self.filter == 'taxids' and\
# (int(tax_num) not in self.tax_ids)) or\
# (self.filter == 'geneids' and\
# (int(gene_num) not in self.gene_ids))):
# continue
# #### end filter
line_counter += 1
pheno_list = []
if phenotype_ids != '-':
# trim any leading/trailing semicolons/commas
phenotype_ids = re.sub(r'^[;,]', '', phenotype_ids)
phenotype_ids = re.sub(r'[;,]$', '', phenotype_ids)
pheno_list = re.split(r'[,;]', phenotype_ids)
if self.testMode:
# get intersection of test disease ids
# and these phenotype_ids
intersect = \
list(
set([str(i)
for i in self.disease_ids]) & set(pheno_list))
if int(gene_num) not in self.gene_ids and\
int(variant_num) not in self.variant_ids and\
len(intersect) < 1:
continue
# TODO may need to switch on assembly to create correct
# assembly/build identifiers
build_id = ':'.join(('NCBIGenome', assembly))
# make the reference genome build
geno.addReferenceGenome(build_id, assembly, tax_id)
allele_type_id = self._map_type_of_allele(allele_type)
bandinbuild_id = None
if str(chr) == '':
# check cytogenic location
if str(cytogenetic_loc).strip() != '':
# use cytogenic location to get the apx location
# oddly, they still put an assembly number even when
# there's no numeric location
if not re.search(r'-', str(cytogenetic_loc)):
band_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)),
tax_num, 'CHR')
geno.addChromosomeInstance(
cytogenetic_loc, build_id, assembly, band_id)
bandinbuild_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)),
assembly, 'MONARCH')
else:
# can't deal with ranges yet
pass
else:
# add the human chromosome class to the graph,
# and add the build-specific version of it
chr_id = makeChromID(str(chr), tax_num, 'CHR')
geno.addChromosomeClass(str(chr), tax_id, tax_label)
geno.addChromosomeInstance(
str(chr), build_id, assembly, chr_id)
chrinbuild_id = makeChromID(str(chr), assembly, 'MONARCH')
seqalt_id = ':'.join(('ClinVarVariant', variant_num))
gene_id = None
# they use -1 to indicate unknown gene
if str(gene_num) != '-1' and str(gene_num) != 'more than 10':
if re.match(r'^Gene:', gene_num):
gene_num = "NCBI" + gene_num
else:
gene_id = ':'.join(('NCBIGene', str(gene_num)))
# FIXME there are some "variants" that are actually haplotypes
# probably will get taken care of when we switch to processing
# the xml for example, variant_num = 38562
# but there's no way to tell if it's a haplotype
# in the csv data so the dbsnp or dbvar
# should probably be primary,
# and the variant num be the vslc,
# with each of the dbsnps being added to it
# TODO clinical significance needs to be mapped to
# a list of terms
# first, make the variant:
f = Feature(seqalt_id, allele_name, allele_type_id)
if start != '-' and start.strip() != '':
f.addFeatureStartLocation(start, chrinbuild_id)
if stop != '-' and stop.strip() != '':
f.addFeatureEndLocation(stop, chrinbuild_id)
f.addFeatureToGraph()
f.addTaxonToFeature(tax_id)
# make the ClinVarVariant the clique leader
model.makeLeader(seqalt_id)
if bandinbuild_id is not None:
f.addSubsequenceOfFeature(bandinbuild_id)
# CHECK - this makes the assumption that there is
# only one affected chromosome per variant what happens with
# chromosomal rearrangement variants?
# shouldn't both chromosomes be here?
# add the hgvs as synonyms
if hgvs_c != '-' and hgvs_c.strip() != '':
model.addSynonym(seqalt_id, hgvs_c)
if hgvs_p != '-' and hgvs_p.strip() != '':
model.addSynonym(seqalt_id, hgvs_p)
# add the dbsnp and dbvar ids as equivalent
if dbsnp_num != '-' and int(dbsnp_num) != -1:
dbsnp_id = 'dbSNP:rs'+str(dbsnp_num)
model.addIndividualToGraph(dbsnp_id, None)
model.addSameIndividual(seqalt_id, dbsnp_id)
if dbvar_num != '-':
dbvar_id = 'dbVar:'+dbvar_num
model.addIndividualToGraph(dbvar_id, None)
model.addSameIndividual(seqalt_id, dbvar_id)
# TODO - not sure if this is right... add as xref?
# the rcv is like the combo of the phenotype with the variant
if rcv_nums != '-':
for rcv_num in re.split(r';', rcv_nums):
rcv_id = 'ClinVar:' + rcv_num
model.addIndividualToGraph(rcv_id, None)
model.addXref(seqalt_id, rcv_id)
if gene_id is not None:
# add the gene
model.addClassToGraph(gene_id, gene_symbol)
# make a variant locus
vl_id = '_'+gene_num+'-'+variant_num
if self.nobnodes:
vl_id = ':'+vl_id
vl_label = allele_name
model.addIndividualToGraph(
vl_id, vl_label, geno.genoparts['variant_locus'])
geno.addSequenceAlterationToVariantLocus(seqalt_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
# some basic reporting
gmatch = re.search(r'\(\w+\)', allele_name)
if gmatch is not None and len(gmatch.groups()) > 0:
logger.info(
"Gene found in allele label, but no id provided: %s",
gmatch.group(1))
elif re.match(r'more than 10', gene_symbol):
logger.info(
"More than 10 genes found; "
"need to process XML to fetch (variant=%d)",
int(variant_num))
else:
logger.info(
"No gene listed for variant %d",
int(variant_num))
# parse the list of "phenotypes" which are diseases.
# add them as an association
# ;GeneReviews:NBK1440,MedGen:C0392514,OMIM:235200,SNOMED CT:35400008;MedGen:C3280096,OMIM:614193;MedGen:CN034317,OMIM:612635;MedGen:CN169374
# the list is both semicolon delimited and comma delimited,
# but i don't know why! some are bad, like:
# Orphanet:ORPHA ORPHA319705,SNOMED CT:49049000
if phenotype_ids != '-':
for phenotype in pheno_list:
m = re.match(
r"(Orphanet:ORPHA(?:\s*ORPHA)?)", phenotype)
if m is not None and len(m.groups()) > 0:
phenotype = re.sub(
m.group(1), 'Orphanet:', phenotype.strip())
elif re.match(r'ORPHA:\d+', phenotype):
phenotype = re.sub(
r'^ORPHA', 'Orphanet', phenotype.strip())
elif re.match(r'Human Phenotype Ontology', phenotype):
phenotype = re.sub(
r'^Human Phenotype Ontology', '',
phenotype.strip())
elif re.match(r'SNOMED CT:\s?', phenotype):
phenotype = re.sub(
r'SNOMED CT:\s?', 'SNOMED:', phenotype.strip())
elif re.match(r'^Gene:', phenotype):
continue
assoc = G2PAssoc(
g, self.name, seqalt_id, phenotype.strip())
assoc.add_association_to_graph()
if other_ids != '-':
id_list = other_ids.split(',')
# process the "other ids" ex:
# CFTR2:F508del,HGMD:CD890142,OMIM Allelic Variant:602421.0001
# TODO make more xrefs
for xrefid in id_list:
prefix = xrefid.split(':')[0].strip()
if prefix == 'OMIM Allelic Variant':
xrefid = 'OMIM:'+xrefid.split(':')[1]
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'HGMD':
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'dbVar' \
and dbvar_num == xrefid.split(':')[1].strip():
pass # skip over this one
elif re.search(r'\s', prefix):
pass
# logger.debug(
# 'xref prefix has a space: %s', xrefid)
else:
# should be a good clean prefix
# note that HGMD variants are in here as Xrefs
# because we can't resolve URIs for them
# logger.info("Adding xref: %s", xrefid)
# gu.addXref(g, seqalt_id, xrefid)
# logger.info("xref prefix to add: %s", xrefid)
pass
if not self.testMode and limit is not None \
and line_counter > limit:
break
logger.info("Finished parsing variants")
return
class OMIA(Source):
"""
This is the parser for the
[Online Mendelian Inheritance in Animals
(OMIA)](http://www.http://omia.angis.org.au),
from which we process inherited disorders, other (single-locus) traits,
and genes in >200 animal species (other than human and mouse and rats).
We generate the omia graph to include the following information:
* genes
* animal taxonomy, and breeds as instances of those taxa
(breeds are akin to "strains" in other taxa)
* animal diseases, along with species-specific subtypes of those diseases
* publications (and their mapping to PMIDs, if available)
* gene-to-phenotype associations (via an anonymous variant-locus
* breed-to-phenotype associations
We make links between OMIA and OMIM in two ways:
1. mappings between OMIA and OMIM are created as OMIA --> hasdbXref OMIM
2. mappings between a breed and OMIA disease are created
to be a model for the mapped OMIM disease,
IF AND ONLY IF it is a 1:1 mapping.
there are some 1:many mappings,
and these often happen if the OMIM item is a gene.
Because many of these species are not covered in
the PANTHER orthology datafiles, we also pull any orthology
relationships from the gene_group files from NCBI.
"""
files = {
'data': {
'file': 'omia.xml.gz',
'url': 'http://omia.angis.org.au/dumps/omia.xml.gz'},
}
def __init__(self):
Source.__init__(self, 'omia')
self.load_bindings()
self.dataset = Dataset(
'omia', 'Online Mendelian Inheritance in Animals',
'http://omia.angis.org.au', None, None,
'http://sydney.edu.au/disclaimer.shtml')
self.id_hash = {
'article': {},
'phene': {},
'breed': {},
'taxon': {},
'gene': {}
}
self.label_hash = {}
self.gu = GraphUtils(curie_map.get())
# used to store the omia to omim phene mappings
self.omia_omim_map = {}
# used to store the unique genes that have phenes
# (for fetching orthology)
self.annotated_genes = set()
self.test_ids = {
'disease': [
'OMIA:001702', 'OMIA:001867', 'OMIA:000478', 'OMIA:000201',
'OMIA:000810', 'OMIA:001400'],
'gene': [
492297, 434, 492296, 3430235, 200685834, 394659996, 200685845,
28713538, 291822383],
'taxon': [9691, 9685, 9606, 9615, 9913, 93934, 37029, 9627, 9825],
# to be filled in during parsing of breed table
# for lookup by breed-associations
'breed': []
}
# to store a map of omia ids and any molecular info
# to write a report for curation
self.stored_omia_mol_gen = {}
self.g = self.graph
self.geno = Genotype(self.g)
return
def fetch(self, is_dl_forced=False):
"""
:param is_dl_forced:
:return:
"""
self.get_files(is_dl_forced)
ncbi = NCBIGene()
# ncbi.fetch()
gene_group = ncbi.files['gene_group']
self.fetch_from_url(
gene_group['url'], '/'.join((ncbi.rawdir, gene_group['file'])),
False)
return
def parse(self, limit=None):
# names of tables to iterate - probably don't need all these:
# Article_Breed, Article_Keyword, Article_Gene, Article_Keyword,
# Article_People, Article_Phene, Articles, Breed, Breed_Phene,
# Genes_gb, Group_Categories, Group_MPO, Inherit_Type, Keywords,
# Landmark, Lida_Links, OMIA_Group, OMIA_author, Omim_Xref, People,
# Phene, Phene_Gene, Publishers, Resources, Species_gb, Synonyms
self.scrub()
if limit is not None:
logger.info("Only parsing first %d rows", limit)
logger.info("Parsing files...")
if self.testOnly:
self.testMode = True
if self.testMode:
self.g = self.testgraph
else:
self.g = self.graph
self.geno = Genotype(self.g)
# we do three passes through the file
# first process species (two others reference this one)
self.process_species(limit)
# then, process the breeds, genes, articles, and other static stuff
self.process_classes(limit)
# next process the association data
self.process_associations(limit)
# process the vertebrate orthology for genes
# that are annotated with phenotypes
ncbi = NCBIGene()
ncbi.add_orthologs_by_gene_group(self.g, self.annotated_genes)
self.load_core_bindings()
self.load_bindings()
logger.info("Done parsing.")
self.write_molgen_report()
return
def scrub(self):
"""
The XML file seems to have mixed-encoding;
we scrub out the control characters
from the file for processing.
:return:
"""
logger.info(
"Scrubbing out the nasty characters that break our parser.")
myfile = '/'.join((self.rawdir, self.files['data']['file']))
tmpfile = '/'.join((self.rawdir, self.files['data']['file']+'.tmp.gz'))
t = gzip.open(tmpfile, 'wb')
du = DipperUtil()
with gzip.open(myfile, 'rb') as f:
filereader = io.TextIOWrapper(f, newline="")
for l in filereader:
l = du.remove_control_characters(l) + '\n'
t.write(l.encode('utf-8'))
t.close()
# move the temp file
logger.info("Replacing the original data with the scrubbed file.")
shutil.move(tmpfile, myfile)
return
# ###################### XML LOOPING FUNCTIONS ##################
def process_species(self, limit):
"""
Loop through the xml file and process the species.
We add elements to the graph, and store the
id-to-label in the label_hash dict.
:param limit:
:return:
"""
myfile = '/'.join((self.rawdir, self.files['data']['file']))
f = gzip.open(myfile, 'rb')
filereader = io.TextIOWrapper(f, newline="")
filereader.readline() # remove the xml declaration line
for event, elem in ET.iterparse(filereader):
# Species ids are == genbank species ids!
self.process_xml_table(
elem, 'Species_gb', self._process_species_table_row, limit)
f.close()
return
def process_classes(self, limit):
"""
Loop through the xml file and process the articles,
breed, genes, phenes, and phenotype-grouping classes.
We add elements to the graph,
and store the id-to-label in the label_hash dict,
along with the internal key-to-external id in the id_hash dict.
The latter are referenced in the association processing functions.
:param limit:
:return:
"""
myfile = '/'.join((self.rawdir, self.files['data']['file']))
f = gzip.open(myfile, 'rb')
filereader = io.TextIOWrapper(f, newline="")
filereader.readline() # remove the xml declaration line
parser = ET.XMLParser(encoding='utf-8')
for event, elem in ET.iterparse(filereader, parser=parser):
self.process_xml_table(
elem, 'Articles', self._process_article_row, limit)
self.process_xml_table(
elem, 'Breed', self._process_breed_row, limit)
self.process_xml_table(
elem, 'Genes_gb', self._process_gene_row, limit)
self.process_xml_table(
elem, 'OMIA_Group', self._process_omia_group_row, limit)
self.process_xml_table(
elem, 'Phene', self._process_phene_row, limit)
self.process_xml_table(
elem, 'Omim_Xref', self._process_omia_omim_map, limit)
f.close()
# post-process the omia-omim associations to filter out the genes
# (keep only phenotypes/diseases)
self.clean_up_omim_genes()
return
def process_associations(self, limit):
"""
Loop through the xml file and process the article-breed, article-phene,
breed-phene, phene-gene associations, and the external links to LIDA.
:param limit:
:return:
"""
myfile = '/'.join((self.rawdir, self.files['data']['file']))
f = gzip.open(myfile, 'rb')
filereader = io.TextIOWrapper(f, newline="")
filereader.readline() # remove the xml declaration line
for event, elem in ET.iterparse(filereader):
self.process_xml_table(
elem, 'Article_Breed', self._process_article_breed_row, limit)
self.process_xml_table(
elem, 'Article_Phene', self._process_article_phene_row, limit)
self.process_xml_table(
elem, 'Breed_Phene', self._process_breed_phene_row, limit)
self.process_xml_table(
elem, 'Lida_Links', self._process_lida_links_row, limit)
self.process_xml_table(
elem, 'Phene_Gene', self._process_phene_gene_row, limit)
self.process_xml_table(
elem, 'Group_MPO', self._process_group_mpo_row, limit)
f.close()
return
# ############ INDIVIDUAL TABLE-LEVEL PROCESSING FUNCTIONS ################
def _process_species_table_row(self, row):
# gb_species_id, sci_name, com_name, added_by, date_modified
tax_id = 'NCBITaxon:'+str(row['gb_species_id'])
sci_name = row['sci_name']
com_name = row['com_name']
if self.testMode and \
(int(row['gb_species_id']) not in self.test_ids['taxon']):
return
self.gu.addClassToGraph(self.g, tax_id, sci_name)
if com_name != '':
self.gu.addSynonym(self.g, tax_id, com_name)
self.label_hash[tax_id] = com_name # for lookup later
else:
self.label_hash[tax_id] = sci_name
return
def _process_breed_row(self, row):
# in test mode, keep all breeds of our test species
if self.testMode and \
(int(row['gb_species_id']) not in self.test_ids['taxon']):
return
# save the breed keys in the test_ids for later processing
self.test_ids['breed'] += [int(row['breed_id'])]
breed_id = self.make_breed_id(row['breed_id'])
self.id_hash['breed'][row['breed_id']] = breed_id
tax_id = 'NCBITaxon:'+str(row['gb_species_id'])
breed_label = row['breed_name']
species_label = self.label_hash.get(tax_id)
if species_label is not None:
breed_label = breed_label + ' ('+species_label+')'
self.gu.addIndividualToGraph(self.g, breed_id, breed_label, tax_id)
self.label_hash[breed_id] = breed_label
return
def _process_phene_row(self, row):
phenotype_id = None
sp_phene_label = row['phene_name']
if sp_phene_label == '':
sp_phene_label = None
if 'omia_id' not in row:
logger.info("omia_id not present for %s", row['phene_id'])
omia_id = self._make_internal_id('phene', phenotype_id)
else:
omia_id = 'OMIA:'+str(row['omia_id'])
if self.testMode and not\
(int(row['gb_species_id']) in self.test_ids['taxon'] and
omia_id in self.test_ids['disease']):
return
# add to internal hash store for later lookup
self.id_hash['phene'][row['phene_id']] = omia_id
descr = row['summary']
if descr == '':
descr = None
# omia label
omia_label = self.label_hash.get(omia_id)
# add the species-specific subclass (TODO please review this choice)
gb_species_id = row['gb_species_id']
if gb_species_id != '':
sp_phene_id = '-'.join((omia_id, gb_species_id))
else:
logger.error(
"No species supplied in species-specific phene table for %s",
omia_id)
return
species_id = 'NCBITaxon:'+str(gb_species_id)
# use this instead
species_label = self.label_hash.get('NCBITaxon:'+gb_species_id)
if sp_phene_label is None and \
omia_label is not None and species_label is not None:
sp_phene_label = ' '.join((omia_label, 'in', species_label))
self.gu.addClassToGraph(
self.g, sp_phene_id, sp_phene_label, omia_id, descr)
# add to internal hash store for later lookup
self.id_hash['phene'][row['phene_id']] = sp_phene_id
self.label_hash[sp_phene_id] = sp_phene_label
# add each of the following descriptions,
# if they are populated, with a tag at the end.
for item in [
'clin_feat', 'history', 'pathology', 'mol_gen', 'control']:
if row[item] is not None and row[item] != '':
self.gu.addDescription(
self.g, sp_phene_id, row[item] + ' ['+item+']')
# if row['symbol'] is not None: # species-specific
# CHECK ME - sometimes spaces or gene labels
# gu.addSynonym(g, sp_phene, row['symbol'])
self.gu.addOWLPropertyClassRestriction(
self.g, sp_phene_id, self.gu.object_properties['in_taxon'],
species_id)
# add inheritance as an association
inheritance_id = self._map_inheritance_term_id(row['inherit'])
if inheritance_id is not None:
assoc = DispositionAssoc(self.name, sp_phene_id, inheritance_id)
assoc.add_association_to_graph(self.g)
if row['characterised'] == 'Yes':
self.stored_omia_mol_gen[omia_id] = {
'mol_gen': row['mol_gen'],
'map_info': row['map_info'],
'species': row['gb_species_id']}
return
def write_molgen_report(self):
import csv
logger.info("Writing G2P report for OMIA")
f = '/'.join((self.outdir, 'omia_molgen_report.txt'))
with open(f, 'w', newline='\n') as csvfile:
writer = csv.writer(csvfile, delimiter='\t')
# write header
h = ['omia_id', 'molecular_description', 'mapping_info', 'species']
writer.writerow(h)
for phene in self.stored_omia_mol_gen:
writer.writerow((str(phene),
self.stored_omia_mol_gen[phene]['mol_gen'],
self.stored_omia_mol_gen[phene]['map_info'],
self.stored_omia_mol_gen[phene]['species']))
logger.info(
"Wrote %d potential G2P descriptions for curation to %s",
len(self.stored_omia_mol_gen), f)
return
def _process_article_row(self, row):
# don't bother in test mode
if self.testMode:
return
iarticle_id = self._make_internal_id('article', row['article_id'])
self.id_hash['article'][row['article_id']] = iarticle_id
rtype = None
if row['journal'] != '':
rtype = Reference.ref_types['journal_article']
r = Reference(iarticle_id, rtype)
if row['title'] is not None:
r.setTitle(row['title'].strip())
if row['year'] is not None:
r.setYear(row['year'])
r.addRefToGraph(self.g)
if row['pubmed_id'] is not None:
pmid = 'PMID:'+str(row['pubmed_id'])
self.id_hash['article'][row['article_id']] = pmid
self.gu.addSameIndividual(self.g, iarticle_id, pmid)
self.gu.addComment(self.g, pmid, iarticle_id)
return
def _process_omia_group_row(self, row):
omia_id = 'OMIA:'+row['omia_id']
if self.testMode and omia_id not in self.test_ids['disease']:
return
group_name = row['group_name']
group_summary = row['group_summary']
disease_id = None
group_category = row.get('group_category')
disease_id = \
self.map_omia_group_category_to_ontology_id(group_category)
if disease_id is not None:
self.gu.addClassToGraph(self.g, disease_id, None)
if disease_id == 'MP:0008762': # embryonic lethal
# add this as a phenotype association
# add embryonic onset
assoc = D2PAssoc(self.name, omia_id, disease_id)
assoc.add_association_to_graph(self.g)
disease_id = None
else:
logger.info(
"No disease superclass defined for %s: %s",
omia_id, group_name)
# default to general disease FIXME this may not be desired
disease_id = 'DOID:4'
if group_summary == '':
group_summary = None
if group_name == '':
group_name = None
self.gu.addClassToGraph(
self.g, omia_id, group_name, disease_id, group_summary)
self.label_hash[omia_id] = group_name
return
def _process_gene_row(self, row):
if self.testMode and row['gene_id'] not in self.test_ids['gene']:
return
gene_id = 'NCBIGene:'+str(row['gene_id'])
self.id_hash['gene'][row['gene_id']] = gene_id
gene_label = row['symbol']
self.label_hash[gene_id] = gene_label
tax_id = 'NCBITaxon:'+str(row['gb_species_id'])
gene_type_id = NCBIGene.map_type_of_gene(row['gene_type'])
self.gu.addClassToGraph(self.g, gene_id, gene_label, gene_type_id)
self.geno.addTaxon(tax_id, gene_id)
return
def _process_article_breed_row(self, row):
# article_id, breed_id, added_by
# don't bother putting these into the test... too many!
# and int(row['breed_id']) not in self.test_ids['breed']:
if self.testMode:
return
article_id = self.id_hash['article'].get(row['article_id'])
breed_id = self.id_hash['breed'].get(row['breed_id'])
# there's some missing data (article=6038). in that case skip
if article_id is not None:
self.gu.addTriple(
self.g, article_id, self.gu.object_properties['is_about'],
breed_id)
else:
logger.warning("Missing article key %s", str(row['article_id']))
return
def _process_article_phene_row(self, row):
"""
Linking articles to species-specific phenes.
:param row:
:return:
"""
# article_id, phene_id, added_by
# look up the article in the hashmap
phenotype_id = self.id_hash['phene'].get(row['phene_id'])
article_id = self.id_hash['article'].get(row['article_id'])
omia_id = self._get_omia_id_from_phene_id(phenotype_id)
if self.testMode and omia_id not in self.test_ids['disease'] \
or phenotype_id is None or article_id is None:
return
# make a triple, where the article is about the phenotype
self.gu.addTriple(
self.g, article_id,
self.gu.object_properties['is_about'], phenotype_id)
return
def _process_breed_phene_row(self, row):
# Linking disorders/characteristic to breeds
# breed_id, phene_id, added_by
breed_id = self.id_hash['breed'].get(row['breed_id'])
phene_id = self.id_hash['phene'].get(row['phene_id'])
# get the omia id
omia_id = self._get_omia_id_from_phene_id(phene_id)
if (self.testMode and not (
omia_id in self.test_ids['disease'] and
int(row['breed_id']) in self.test_ids['breed']) or
breed_id is None or phene_id is None):
return
# FIXME we want a different relationship here
assoc = G2PAssoc(
self.name, breed_id, phene_id,
self.gu.object_properties['has_phenotype'])
assoc.add_association_to_graph(self.g)
# add that the breed is a model of the human disease
# use the omia-omim mappings for this
# we assume that we have already scrubbed out the genes
# from the omim list, so we can make the model associations here
omim_ids = self.omia_omim_map.get(omia_id)
eco_id = "ECO:0000214" # biological aspect of descendant evidence
if omim_ids is not None and len(omim_ids) > 0:
if len(omim_ids) > 1:
logger.info(
"There's 1:many omia:omim mapping: %s, %s",
omia_id, str(omim_ids))
for i in omim_ids:
assoc = G2PAssoc(
self.name, breed_id, i,
self.gu.object_properties['model_of'])
assoc.add_evidence(eco_id)
assoc.add_association_to_graph(self.g)
aid = assoc.get_association_id()
breed_label = self.label_hash.get(breed_id)
if breed_label is None:
breed_label = "this breed"
m = re.search(r'\((.*)\)', breed_label)
if m:
sp_label = m.group(1)
else:
sp_label = ''
phene_label = self.label_hash.get(phene_id)
if phene_label is None:
phene_label = "phenotype"
elif phene_label.endswith(sp_label):
# some of the labels we made already include the species;
# remove it to make a cleaner desc
phene_label = re.sub(r' in '+sp_label, '', phene_label)
desc = ' '.join(
("High incidence of", phene_label, "in", breed_label,
"suggests it to be a model of disease", i + "."))
self.gu.addDescription(self.g, aid, desc)
return
def _process_lida_links_row(self, row):
# lidaurl, omia_id, added_by
omia_id = 'OMIA:'+row['omia_id']
lidaurl = row['lidaurl']
if self.testMode and omia_id not in self.test_ids['disease']:
return
self.gu.addXref(self.g, omia_id, lidaurl, True)
return
def _process_phene_gene_row(self, row):
gene_id = self.id_hash['gene'].get(row['gene_id'])
phene_id = self.id_hash['phene'].get(row['phene_id'])
omia_id = self._get_omia_id_from_phene_id(phene_id)
if self.testMode and not (
omia_id in self.test_ids['disease'] and
row['gene_id'] in self.test_ids['gene']) or\
gene_id is None or phene_id is None:
return
# occasionally some phenes are missing! (ex: 406)
if phene_id is None:
logger.warning("Phene id %s is missing", str(row['phene_id']))
return
gene_label = self.label_hash[gene_id]
# some variant of gene_id has phenotype d
vl = '_'+re.sub(r'NCBIGene:', '', str(gene_id)) + 'VL'
if self.nobnodes:
vl = ':'+vl
self.geno.addAllele(vl, 'some variant of ' + gene_label)
self.geno.addAlleleOfGene(vl, gene_id)
assoc = G2PAssoc(self.name, vl, phene_id)
assoc.add_association_to_graph(self.g)
# add the gene id to the set of annotated genes
# for later lookup by orthology
self.annotated_genes.add(gene_id)
return
def _process_omia_omim_map(self, row):
"""
Links OMIA groups to OMIM equivalents.
:param row:
:return:
"""
# omia_id, omim_id, added_by
omia_id = 'OMIA:'+row['omia_id']
omim_id = 'OMIM:'+row['omim_id']
# also store this for use when we say that a given animal is
# a model of a disease
if omia_id not in self.omia_omim_map:
self.omia_omim_map[omia_id] = set()
self.omia_omim_map[omia_id].add(omim_id)
if self.testMode and omia_id not in self.test_ids['disease']:
return
self.gu.addXref(self.g, omia_id, omim_id)
return
def map_omia_group_category_to_ontology_id(self, category_num):
"""
Using the category number in the OMIA_groups table,
map them to a disease id.
This may be superceeded by other MONDO methods.
Platelet disorders will be more specific once
https://github.com/obophenotype/human-disease-ontology/issues/46
is fulfilled.
:param category_num:
:return:
"""
category_map = {
1: 'DOID:0014667', # Inborn error of metabolism
2: 'MESH:D004392', # Dwarfism
3: 'DOID:1682', # congenital heart disease
4: 'DOID:74', # blood system disease
5: 'DOID:3211', # lysosomal storage disease
6: 'DOID:16', # integumentary system disease
# --> retinal degeneration ==> OMIA:000830
7: 'DOID:8466', # progressive retinal atrophy
8: 'DOID:0050572', # Cone–rod dystrophy
9: 'MESH:C536122', # stationary night blindness
10: 'Orphanet:98553', # developmental retinal disorder
11: 'DOID:5679', # retinal disorder
12: 'Orphanet:90771', # Disorder of Sex Development
# - what to do about this one?
13: 'MP:0008762', # embryonic lethal
# - not sure what to do with this
14: None, # blood group
# FIXME make me more specific
15: 'DOID:2218', # intrinsic platelet disorder
# FIXME make me more specific
16: 'DOID:2218', # extrinsic platelet disorder
17: None # transgenic ???
}
disease_id = None
if category_num is not None and int(category_num) in category_map:
disease_id = category_map.get(int(category_num))
logger.info(
"Found %s for category %s", str(disease_id), str(category_num))
else:
logger.info(
"There's a group category I don't know anything about: %s",
str(category_num))
return disease_id
def _process_group_mpo_row(self, row):
"""
Make OMIA to MP associations
:param row:
:return:
"""
omia_id = 'OMIA:'+row['omia_id']
mpo_num = int(row['MPO_no'])
mpo_id = 'MP:'+str(mpo_num).zfill(7)
assoc = D2PAssoc(self.name, omia_id, mpo_id)
assoc.add_association_to_graph(self.g)
return
def clean_up_omim_genes(self):
omim = OMIM()
# get all the omim ids
allomimids = set()
for omia in self.omia_omim_map:
allomimids.update(self.omia_omim_map[omia])
entries_that_are_phenotypes = omim.process_entries(
list(allomimids), filter_keep_phenotype_entry_ids, None, None)
logger.info(
"Filtered out %d/%d entries that are genes or features",
len(allomimids)-len(entries_that_are_phenotypes), len(allomimids))
# now iterate again and remove those non-phenotype ids
removed_count = 0
for omia in self.omia_omim_map:
ids = self.omia_omim_map[omia]
cleanids = set()
for i in ids:
if i in entries_that_are_phenotypes:
cleanids.add(i)
else:
removed_count += 1 # keep track of how many we've removed
self.omia_omim_map[omia] = cleanids
logger.info(
"Removed %d omim ids from the omia-to-omim map", removed_count)
return
def _make_internal_id(self, prefix, key):
iid = '_'+''.join(('omia', prefix, 'key', str(key)))
if self.nobnodes:
iid = ':'+iid
return iid
def make_breed_id(self, key):
breed_id = 'OMIA-breed:'+str(key)
return breed_id
@staticmethod
def _get_omia_id_from_phene_id(phene_id):
omia_id = None
if phene_id is not None:
m = re.match(r'OMIA:\d+', str(phene_id))
if m:
omia_id = m.group(0)
return omia_id
@staticmethod
def _map_inheritance_term_id(inheritance_symbol):
inherit_map = {
'A': None, # Autosomal
'ACD': 'GENO:0000143', # Autosomal co-dominant
'ADV': None, # autosomal dominant with variable expressivity
'AID': 'GENO:0000259', # autosomal incompletely dominant
'ASD': 'GENO:0000145', # autosomal semi-dominant
# autosomal recessive, semi-lethal
# using generic autosomal recessive
'ASL': 'GENO:0000150',
'D': 'GENO:0000147', # autosomal dominant
'M': None, # multifactorial
'MAT': None, # Maternal
# probably autosomal recessive
# using generic autosomal recessive
'PR': 'GENO:0000150',
'R': 'GENO:0000150', # Autosomal Recessive
# Recessive Embryonic Lethal
# using plain recessive
'REL': 'GENO:0000148',
# Autosomal Recessive Lethal
# using plain autosomal recessive
'RL': 'GENO:0000150',
'S': 'GENO:0000146', # Sex-linked <--using allosomal dominant
'SLi': None, # Sex-limited
'UD': 'GENO:0000144', # Dominant
'X': None, # x-linked # HP:0001417 ?
# X-linked Dominant <-- temp using allosomal dominant FIXME
'XLD': 'GENO:0000146',
# X-linked Recessive <-- temp using allosomal recessive FIXME
'XLR': 'GENO:0000149',
'Y': None, # Y-linked
'Z': None, # Z-linked
# Z-linked recessive <-- temp using allosomal recessive FIXME
'ZR': 'GENO:0000149',
'999': None, # Z-linked incompletely dominant
}
inheritance_id = inherit_map.get(inheritance_symbol)
if inheritance_id is None and inheritance_symbol is not None:
logger.warning(
"No inheritance id is mapped for %s", inheritance_symbol)
return inheritance_id
def getTestSuite(self):
import unittest
from tests.test_omia import OMIATestCase
test_suite = unittest.TestLoader().loadTestsFromTestCase(OMIATestCase)
return test_suite
def _process_data(self, raw, limit=None):
logger.info("Processing Data from %s", raw)
gu = GraphUtils(curie_map.get())
if self.testMode:
g = self.testgraph
else:
g = self.graph
geno = Genotype(g)
line_counter = 0
gu.loadAllProperties(g)
gu.loadObjectProperties(g, geno.object_properties)
# Add the taxon as a class
taxon_id = 'NCBITaxon:10090' # map to Mus musculus
gu.addClassToGraph(g, taxon_id, None)
# with open(raw, 'r', encoding="utf8") as csvfile:
with gzip.open(raw, 'rt') as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
next(filereader, None) # skip the header row
for row in filereader:
line_counter += 1
(marker_accession_id, marker_symbol, phenotyping_center,
colony, sex, zygosity, allele_accession_id, allele_symbol,
allele_name, strain_accession_id, strain_name, project_name,
project_fullname, pipeline_name, pipeline_stable_id,
procedure_stable_id, procedure_name, parameter_stable_id,
parameter_name, top_level_mp_term_id, top_level_mp_term_name,
mp_term_id, mp_term_name, p_value, percentage_change,
effect_size, statistical_method, resource_name) = row
if self.testMode and marker_accession_id not in self.test_ids:
continue
# ##### cleanup some of the identifiers ######
zygosity_id = self._map_zygosity(zygosity)
# colony ids sometimes have <> in them, spaces,
# or other non-alphanumerics and break our system;
# replace these with underscores
colony_id = '_'+re.sub(r'\W+', '_', colony)
if self.nobnodes:
colony_id = ':'+colony_id
if not re.match(r'MGI', allele_accession_id):
allele_accession_id = \
'_IMPC-'+re.sub(r':', '', allele_accession_id)
if self.nobnodes:
allele_accession_id = ':'+allele_accession_id
if re.search(r'EUROCURATE', strain_accession_id):
# the eurocurate links don't resolve at IMPC
strain_accession_id = '_'+strain_accession_id
if self.nobnodes:
strain_accession_id = ':'+strain_accession_id
elif not re.match(r'MGI', strain_accession_id):
logger.info(
"Found a strange strain accession...%s",
strain_accession_id)
strain_accession_id = 'IMPC:'+strain_accession_id
######################
# first, add the marker and variant to the graph as with MGI,
# the allele is the variant locus. IF the marker is not known,
# we will call it a sequence alteration. otherwise,
# we will create a BNode for the sequence alteration.
sequence_alteration_id = variant_locus_id = None
variant_locus_name = sequence_alteration_name = None
# extract out what's within the <> to get the symbol
if re.match(r'.*<.*>', allele_symbol):
sequence_alteration_name = \
re.match(r'.*<(.*)>', allele_symbol).group(1)
else:
sequence_alteration_name = allele_symbol
if marker_accession_id is not None and \
marker_accession_id == '':
logger.warning(
"Marker unspecified on row %d", line_counter)
marker_accession_id = None
if marker_accession_id is not None:
variant_locus_id = allele_accession_id
variant_locus_name = allele_symbol
variant_locus_type = geno.genoparts['variant_locus']
geno.addGene(marker_accession_id, marker_symbol,
geno.genoparts['gene'])
geno.addAllele(variant_locus_id, variant_locus_name,
variant_locus_type, None)
geno.addAlleleOfGene(variant_locus_id, marker_accession_id)
sequence_alteration_id = \
'_seqalt'+re.sub(r':', '', allele_accession_id)
if self.nobnodes:
sequence_alteration_id = ':'+sequence_alteration_id
geno.addSequenceAlterationToVariantLocus(
sequence_alteration_id, variant_locus_id)
else:
sequence_alteration_id = allele_accession_id
# IMPC contains targeted mutations with either gene traps,
# knockouts, insertion/intragenic deletions.
# but I don't really know what the SeqAlt is here,
# so I don't add it.
geno.addSequenceAlteration(sequence_alteration_id,
sequence_alteration_name)
# ############# BUILD THE COLONY #############
# First, let's describe the colony that the animals come from
# The Colony ID refers to the ES cell clone
# used to generate a mouse strain.
# Terry sez: we use this clone ID to track
# ES cell -> mouse strain -> mouse phenotyping.
# The same ES clone maybe used at multiple centers,
# so we have to concatenate the two to have a unique ID.
# some useful reading about generating mice from ES cells:
# http://ki.mit.edu/sbc/escell/services/details
# here, we'll make a genotype
# that derives from an ES cell with a given allele.
# the strain is not really attached to the colony.
# the colony/clone is reflective of the allele,
# with unknown zygosity
stem_cell_class = 'ERO:0002002'
gu.addIndividualToGraph(g, colony_id, colony, stem_cell_class)
# vslc of the colony has unknown zygosity
# note that we will define the allele
# (and it's relationship to the marker, etc.) later
# FIXME is it really necessary to create this vslc
# when we always know it's unknown zygosity?
vslc_colony = \
'_'+allele_accession_id+geno.zygosity['indeterminate']
vslc_colony = re.sub(r':', '', vslc_colony)
if self.nobnodes:
vslc_colony = ':'+vslc_colony
vslc_colony_label = allele_symbol+'/<?>'
# for ease of reading, we make the colony genotype variables.
# in the future, it might be desired to keep the vslcs
colony_genotype_id = vslc_colony
colony_genotype_label = vslc_colony_label
geno.addGenotype(colony_genotype_id, colony_genotype_label)
geno.addParts(allele_accession_id, colony_genotype_id,
geno.object_properties['has_alternate_part'])
geno.addPartsToVSLC(
vslc_colony, allele_accession_id, None,
geno.zygosity['indeterminate'],
geno.object_properties['has_alternate_part'])
gu.addTriple(
g, colony_id,
geno.object_properties['has_genotype'],
colony_genotype_id)
# ########## BUILD THE ANNOTATED GENOTYPE ##########
# now, we'll build the genotype of the individual that derives
# from the colony/clone genotype that is attached to
# phenotype = colony_id + strain + zygosity + sex
# (and is derived from a colony)
# this is a sex-agnostic genotype
genotype_id = \
self.make_id(
(colony_id + phenotyping_center + zygosity +
strain_accession_id))
geno.addSequenceDerivesFrom(genotype_id, colony_id)
# build the VSLC of the sex-agnostic genotype
# based on the zygosity
allele1_id = allele_accession_id
allele2_id = allele2_rel = None
allele1_label = allele_symbol
allele2_label = '<?>'
# Making VSLC labels from the various parts,
# can change later if desired.
if zygosity == 'heterozygote':
allele2_label = re.sub(r'<.*', '<+>', allele1_label)
allele2_id = None
elif zygosity == 'homozygote':
allele2_label = allele1_label
allele2_id = allele1_id
allele2_rel = geno.object_properties['has_alternate_part']
elif zygosity == 'hemizygote':
allele2_label = re.sub(r'<.*', '<0>', allele1_label)
allele2_id = None
elif zygosity == 'not_applicable':
allele2_label = re.sub(r'<.*', '<?>', allele1_label)
allele2_id = None
else:
logger.warning("found unknown zygosity %s", zygosity)
break
vslc_name = '/'.join((allele1_label, allele2_label))
# Add the VSLC
vslc_id = '_' + '-'.join((marker_accession_id,
allele_accession_id, zygosity))
vslc_id = re.sub(r':', '', vslc_id)
if self.nobnodes:
vslc_id = ':'+vslc_id
gu.addIndividualToGraph(
g, vslc_id, vslc_name,
geno.genoparts['variant_single_locus_complement'])
geno.addPartsToVSLC(
vslc_id, allele1_id, allele2_id, zygosity_id,
geno.object_properties['has_alternate_part'],
allele2_rel)
# add vslc to genotype
geno.addVSLCtoParent(vslc_id, genotype_id)
# note that the vslc is also the gvc
gu.addType(
g, vslc_id,
Genotype.genoparts['genomic_variation_complement'])
# Add the genomic background
# create the genomic background id and name
if strain_accession_id != '':
genomic_background_id = strain_accession_id
else:
genomic_background_id = None
genotype_name = vslc_name
if genomic_background_id is not None:
geno.addGenotype(
genomic_background_id, strain_name,
geno.genoparts['genomic_background'])
# make a phenotyping-center-specific strain
# to use as the background
pheno_center_strain_label = \
strain_name + '/' + phenotyping_center
pheno_center_strain_id = \
'-'.join((re.sub(r':', '', genomic_background_id),
re.sub(r'\s', '_', phenotyping_center)))
if not re.match(r'^_', pheno_center_strain_id):
pheno_center_strain_id = '_'+pheno_center_strain_id
if self.nobnodes:
pheno_center_strain_id = ':'+pheno_center_strain_id
geno.addGenotype(pheno_center_strain_id,
pheno_center_strain_label,
geno.genoparts['genomic_background'])
geno.addSequenceDerivesFrom(pheno_center_strain_id,
genomic_background_id)
# Making genotype labels from the various parts,
# can change later if desired.
# since the genotype is reflective of the place
# it got made, should put that in to disambiguate
genotype_name = \
genotype_name+' ['+pheno_center_strain_label+']'
geno.addGenomicBackgroundToGenotype(
pheno_center_strain_id, genotype_id)
geno.addTaxon(pheno_center_strain_id, taxon_id)
# this is redundant, but i'll keep in in for now
geno.addSequenceDerivesFrom(genotype_id, colony_id)
genotype_name += '['+colony+']'
geno.addGenotype(genotype_id, genotype_name)
# Make the sex-qualified genotype,
# which is what the phenotype is associated with
sex_qualified_genotype_id = \
self.make_id(
(colony_id + phenotyping_center + zygosity +
strain_accession_id+sex))
sex_qualified_genotype_label = genotype_name+' ('+sex+')'
if sex == 'male':
sq_type_id = geno.genoparts['male_genotype']
elif sex == 'female':
sq_type_id = geno.genoparts['female_genotype']
else:
sq_type_id = geno.genoparts['sex_qualified_genotype']
geno.addGenotype(
sex_qualified_genotype_id,
sex_qualified_genotype_label, sq_type_id)
geno.addParts(
genotype_id, sex_qualified_genotype_id,
geno.object_properties['has_alternate_part'])
if genomic_background_id is not None and \
genomic_background_id != '':
# Add the taxon to the genomic_background_id
geno.addTaxon(taxon_id, genomic_background_id)
else:
# add it as the genomic background
geno.addTaxon(taxon_id, genotype_id)
# ############# BUILD THE G2P ASSOC #############
# from an old email dated July 23 2014:
# Phenotypes associations are made to
# imits colony_id+center+zygosity+gender
phenotype_id = mp_term_id
# it seems that sometimes phenotype ids are missing.
# indicate here
if phenotype_id is None or phenotype_id == '':
logger.warning(
"No phenotype id specified for row %d: %s",
line_counter, str(row))
continue
# experimental_phenotypic_evidence This was used in ZFIN
eco_id = "ECO:0000059"
# the association comes as a result of a g2p from
# a procedure in a pipeline at a center and parameter tested
assoc = G2PAssoc(self.name, sex_qualified_genotype_id,
phenotype_id)
assoc.add_evidence(eco_id)
# assoc.set_score(float(p_value))
# TODO add evidence instance using
# pipeline_stable_id +
# procedure_stable_id +
# parameter_stable_id
assoc.add_association_to_graph(g)
assoc_id = assoc.get_association_id()
# add a free-text description
description = \
' '.join((mp_term_name, 'phenotype determined by',
phenotyping_center, 'in an',
procedure_name, 'assay where',
parameter_name.strip(),
'was measured with an effect_size of',
str(round(float(effect_size), 5)),
'(p =', "{:.4e}".format(float(p_value)), ').'))
gu.addDescription(g, assoc_id, description)
# TODO add provenance information
# resource_id = resource_name
# assoc.addSource(g, assoc_id, resource_id)
if not self.testMode and \
limit is not None and line_counter > limit:
break
gu.loadProperties(g, G2PAssoc.object_properties, gu.OBJPROP)
gu.loadProperties(g, G2PAssoc.annotation_properties, gu.ANNOTPROP)
gu.loadProperties(g, G2PAssoc.datatype_properties, gu.DATAPROP)
return
class CTD(Source):
"""
The Comparative Toxicogenomics Database (CTD) includes curated data describing cross-species chemical–gene/protein
interactions and chemical– and gene–disease associations to illuminate molecular mechanisms underlying variable
susceptibility and environmentally influenced diseases.
Here, we fetch, parse, and convert data from CTD into triples, leveraging only the associations based on
DIRECT evidence (not using the inferred associations). We currently process the following associations:
* chemical-disease
* gene-pathway
* gene-disease
CTD curates relationships between genes and chemicals/diseases with marker/mechanism and/or therapeutic.
Unfortunately, we cannot disambiguate between marker (gene expression) and mechanism (causation)
for these associations. Therefore, we are left to relate these simply by "marker".
CTD also pulls in genes and pathway membership from KEGG and REACTOME. We create groups of these following
the pattern that the specific pathway is a subclass of 'cellular process' (a go process), and
the gene is "involved in" that process.
For diseases, we preferentially use OMIM identifiers when they can be used uniquely over MESH. Otherwise,
we use MESH ids.
Note that we scrub the following identifiers and their associated data:
* REACT:116125 - generic disease class
* MESH:D004283 - dog diseases
* MESH:D004195 - disease models, animal
* MESH:D030342 - genetic diseases, inborn
* MESH:D040181 - genetic dieases, x-linked
* MESH:D020022 - genetic predisposition to a disease
"""
files = {
'chemical_disease_interactions': {
'file': 'CTD_chemicals_diseases.tsv.gz',
'url': 'http://ctdbase.org/reports/CTD_chemicals_diseases.tsv.gz'
},
'gene_pathway': {
'file': 'CTD_genes_pathways.tsv.gz',
'url': 'http://ctdbase.org/reports/CTD_genes_pathways.tsv.gz'
},
'gene_disease': {
'file': 'CTD_genes_diseases.tsv.gz',
'url': 'http://ctdbase.org/reports/CTD_genes_diseases.tsv.gz'
}
}
static_files = {
'publications': {'file': 'CTD_curated_references.tsv'}
}
def __init__(self):
Source.__init__(self, 'ctd')
self.dataset = Dataset('ctd', 'CTD', 'http://ctdbase.org', None, 'http://ctdbase.org/about/legal.jsp')
if 'test_ids' not in config.get_config() or 'gene' not in config.get_config()['test_ids']:
logger.warn("not configured with gene test ids.")
self.test_geneids = []
else:
self.test_geneids = config.get_config()['test_ids']['gene']
if 'test_ids' not in config.get_config() or 'disease' not in config.get_config()['test_ids']:
logger.warn("not configured with disease test ids.")
self.test_diseaseids = []
else:
self.test_diseaseids = config.get_config()['test_ids']['disease']
self.gu = GraphUtils(curie_map.get())
return
def fetch(self, is_dl_forced=False):
"""
Override Source.fetch()
Fetches resources from CTD using the CTD.files dictionary
Args:
:param is_dl_forced (bool): Force download
Returns:
:return None
"""
self.get_files(is_dl_forced)
self._fetch_disambiguating_assoc()
# consider creating subsets of the files that only have direct annotations (not inferred)
return
def parse(self, limit=None):
"""
Override Source.parse()
Parses version and interaction information from CTD
Args:
:param limit (int, optional) limit the number of rows processed
Returns:
:return None
"""
if limit is not None:
logger.info("Only parsing first %d rows", limit)
logger.info("Parsing files...")
# pub_map = dict()
# file_path = '/'.join((self.rawdir,
# self.static_files['publications']['file']))
# if os.path.exists(file_path) is True:
# pub_map = self._parse_publication_file(
# self.static_files['publications']['file']
# )
if self.testOnly:
self.testMode = True
if self.testMode:
self.g = self.testgraph
else:
self.g = self.graph
self.geno = Genotype(self.g)
self.path = Pathway(self.g, self.nobnodes)
self._parse_ctd_file(limit, self.files['chemical_disease_interactions']['file'])
self._parse_ctd_file(limit, self.files['gene_pathway']['file'])
self._parse_ctd_file(limit, self.files['gene_disease']['file'])
self._parse_curated_chem_disease(limit)
self.gu.loadAllProperties(self.g)
# self.gu.loadProperties(self.g, self.REL_MAP, self.gu.OBJPROP)
self.load_bindings()
logger.info("Done parsing files.")
return
def _parse_ctd_file(self, limit, file):
"""
Parses files in CTD.files dictionary
Args:
:param limit (int): limit the number of rows processed
:param file (str): file name (must be defined in CTD.file)
Returns:
:return None
"""
row_count = 0
version_pattern = re.compile('^# Report created: (.+)$')
is_versioned = False
file_path = '/'.join((self.rawdir, file))
with gzip.open(file_path, 'rt') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for row in reader:
# Scan the header lines until we get the version
# There is no official version sp we are using
# the upload timestamp instead
if is_versioned is False:
match = re.match(version_pattern, ' '.join(row))
if match:
version = re.sub(r'\s|:', '-', match.group(1))
# TODO convert this timestamp to a proper timestamp
self.dataset.setVersion(version)
is_versioned = True
elif re.match('^#', ' '.join(row)):
pass
else:
row_count += 1
if file == self.files['chemical_disease_interactions']['file']:
self._process_interactions(row)
elif file == self.files['gene_pathway']['file']:
self._process_pathway(row)
elif file == self.files['gene_disease']['file']:
self._process_disease2gene(row)
if not self.testMode and limit is not None and row_count >= limit:
break
return
def _process_pathway(self, row):
"""
Process row of CTD data from CTD_genes_pathways.tsv.gz
and generate triples
Args:
:param row (list): row of CTD data
Returns:
:return None
"""
self._check_list_len(row, 4)
(gene_symbol, gene_id, pathway_name, pathway_id) = row
if self.testMode and (int(gene_id) not in self.test_geneids):
return
entrez_id = 'NCBIGene:'+gene_id
if re.match('REACT:116125', pathway_id):
# skipping this one, as it is generic "Disease"
return
# convert KEGG pathway ids... KEGG:12345 --> KEGG-path:map12345
if re.match('KEGG', pathway_id):
pathway_id = re.sub('KEGG:', 'KEGG-path:map', pathway_id)
self.gu.addClassToGraph(self.graph, entrez_id, None) # just in case, add it as a class
self.path.addPathway(pathway_id, pathway_name)
self.path.addGeneToPathway(pathway_id, entrez_id)
return
def _fetch_disambiguating_assoc(self):
"""
For any of the items in the chemical-disease association file that have ambiguous association types
we fetch the disambiguated associations using the batch query API, and store these in a file.
Elsewhere, we can loop through the file and create the appropriate associations.
:return:
"""
disambig_file = '/'.join((self.rawdir, self.static_files['publications']['file']))
assoc_file = '/'.join((self.rawdir, self.files['chemical_disease_interactions']['file']))
# check if there is a local association file, and download if it's dated later than the original intxn file
if os.path.exists(disambig_file):
dfile_dt = os.stat(disambig_file)
afile_dt = os.stat(assoc_file)
if dfile_dt < afile_dt:
logger.info("Local disambiguating file date < chem-disease assoc file. Downloading...")
else:
logger.info("Local disambiguating file date > chem-disease assoc file. Skipping download.")
return
all_pubs = set()
dual_evidence = re.compile('^marker\/mechanism\|therapeutic$')
# first get all the unique publications
with gzip.open(assoc_file, 'rt') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for row in reader:
if re.match('^#', ' '.join(row)):
continue
self._check_list_len(row, 10)
(chem_name, chem_id, cas_rn, disease_name, disease_id, direct_evidence,
inferred_gene_symbol, inference_score, omim_ids, pubmed_ids) = row
if direct_evidence == '' or not re.match(dual_evidence, direct_evidence):
continue
if pubmed_ids is not None and pubmed_ids != '':
all_pubs.update(set(re.split('\|', pubmed_ids)))
sorted_pubs = sorted(list(all_pubs))
# now in batches of 4000, we fetch the chemical-disease associations
batch_size = 4000
params = {
'inputType': 'reference',
'report': 'diseases_curated',
'format': 'tsv',
'action': 'Download'
}
url = 'http://ctdbase.org/tools/batchQuery.go?q'
start = 0
end = min((batch_size, len(all_pubs))) # get them in batches of 4000
with open(disambig_file, 'wb') as f:
while start < len(sorted_pubs):
params['inputTerms'] = '|'.join(sorted_pubs[start:end])
# fetch the data from url
logger.info('fetching %d (%d-%d) refs: %s', len(re.split('\|', params['inputTerms'])),
start, end, params['inputTerms'])
data = urllib.parse.urlencode(params)
encoding = 'utf-8'
binary_data = data.encode(encoding)
req = urllib.request.Request(url, binary_data)
resp = urllib.request.urlopen(req)
f.write(resp.read())
start = end
end = min((start+batch_size, len(sorted_pubs)))
return
def _process_interactions(self, row):
"""
Process row of CTD data from CTD_chemicals_diseases.tsv.gz
and generate triples.
Only create associations based on direct evidence (not using the inferred-via-gene),
and unambiguous relationships. (Ambiguous ones will be processed in the sister method using the
disambiguated file). There are no OMIM ids for diseases in these cases, so we associate with only
the mesh disease ids.
Args:
:param row (list): row of CTD data
Returns:
:return None
"""
self._check_list_len(row, 10)
(chem_name, chem_id, cas_rn, disease_name, disease_id, direct_evidence,
inferred_gene_symbol, inference_score, omim_ids, pubmed_ids) = row
if direct_evidence == '':
return
evidence_pattern = re.compile('^therapeutic|marker\/mechanism$')
# dual_evidence = re.compile('^marker\/mechanism\|therapeutic$')
# filter on those diseases that are mapped to omim ids in the test set
intersect = list(set(['OMIM:'+str(i) for i in omim_ids.split('|')]+[disease_id]) & set(self.test_diseaseids))
if self.testMode and len(intersect) < 1:
return
chem_id = 'MESH:'+chem_id
reference_list = self._process_pubmed_ids(pubmed_ids)
if re.match(evidence_pattern, direct_evidence):
rel_id = self._get_relationship_id(direct_evidence)
self.gu.addClassToGraph(self.g, chem_id, chem_name)
self.gu.addClassToGraph(self.g, disease_id, None)
self._make_association(chem_id, disease_id, rel_id, reference_list)
else:
# there's dual evidence, but haven't mapped the pubs
pass
# logger.debug("Dual evidence for %s (%s) and %s (%s)", chem_name, chem_id, disease_name, disease_id)
return
def _process_disease2gene(self, row):
"""
Here, we process the disease-to-gene associations.
Note that we ONLY process direct associations (not inferred through chemicals).
Furthermore, we also ONLY process "marker/mechanism" associations.
We preferentially utilize OMIM identifiers over MESH identifiers for disease/phenotype.
Therefore, if a single OMIM id is listed under the "omim_ids" list, we will choose this over any
MeSH id that might be listed as the disease_id.
If multiple OMIM ids are listed in the omim_ids column, we toss this for now. (Mostly, we are not sure
what to do with this information.)
We associate "some variant of gene X" with the phenotype, rather than the gene directly.
We also pull in the MeSH labels here (but not OMIM) to ensure that we have them (as they may not be
brought in separately).
:param row:
:return:
"""
# if self.testMode:
# g = self.testgraph
# else:
# g = self.graph
# self._check_list_len(row, 9)
# geno = Genotype(g)
# gu = GraphUtils(curie_map.get())
(gene_symbol, gene_id, disease_name, disease_id, direct_evidence,
inference_chemical_name, inference_score, omim_ids, pubmed_ids) = row
# we only want the direct associations; skipping inferred for now
if direct_evidence == '' or direct_evidence != 'marker/mechanism':
return
# scrub some of the associations... it seems odd to link human genes to the following "diseases"
diseases_to_scrub = [
'MESH:D004283', # dog diseases
'MESH:D004195', # disease models, animal
'MESH:D030342', # genetic diseases, inborn
'MESH:D040181', # genetic dieases, x-linked
'MESH:D020022' # genetic predisposition to a disease
]
if disease_id in diseases_to_scrub:
logger.info("Skipping association between NCBIGene:%s and %s", str(gene_id), disease_id)
return
intersect = list(set(['OMIM:'+str(i) for i in omim_ids.split('|')]+[disease_id]) & set(self.test_diseaseids))
if self.testMode and (int(gene_id) not in self.test_geneids or len(intersect) < 1):
return
# there are three kinds of direct evidence: (marker/mechanism | marker/mechanism|therapeutic | therapeutic)
# we are only using the "marker/mechanism" for now
# TODO what does it mean for a gene to be therapeutic for disease? a therapeutic target?
gene_id = 'NCBIGene:'+gene_id
preferred_disease_id = disease_id
if omim_ids is not None and omim_ids != '':
omim_id_list = re.split('\|', omim_ids)
# If there is only one OMIM ID for the Disease ID or in the omim_ids list,
# use the OMIM ID preferentially over any MeSH ID.
if re.match('OMIM:.*', disease_id):
if len(omim_id_list) > 1:
# the disease ID is an OMIM ID and there is more than one OMIM entry in omim_ids.
# Currently no entries satisfy this condition
pass
elif disease_id != ('OMIM:'+omim_ids):
# the disease ID is an OMIM ID and there is only one non-equiv OMIM entry in omim_ids
# we preferentially use the disease_id here
logger.warn("There may be alternate identifier for %s: %s", disease_id, omim_ids)
# TODO: What should be done with the alternate disease IDs?
else:
if len(omim_id_list) == 1:
# the disease ID is not an OMIM ID and there is only one OMIM entry in omim_ids.
preferred_disease_id = 'OMIM:'+omim_ids
elif len(omim_id_list) > 1:
# This is when the disease ID is not an OMIM ID and there is more than one OMIM entry in omim_ids.
pass
# we actually want the association between the gene and the disease to be via an alternate locus
# not the "wildtype" gene itself.
# so we make an anonymous alternate locus, and put that in the association.
alt_locus = '_'+gene_id+'-'+preferred_disease_id+'VL'
alt_locus = re.sub(':', '', alt_locus) # can't have colons in the bnodes
if self.nobnodes:
alt_locus = ':'+alt_locus
alt_label = 'some variant of '+gene_symbol+' that is '+direct_evidence+' for '+disease_name
self.gu.addIndividualToGraph(self.g, alt_locus, alt_label, self.geno.genoparts['variant_locus'])
self.gu.addClassToGraph(self.g, gene_id, None) # assume that the label gets added elsewhere
self.geno.addAlleleOfGene(alt_locus, gene_id)
# not sure if MESH is getting added separately. adding labels here for good measure
dlabel = None
if re.match('MESH', preferred_disease_id):
dlabel = disease_name
self.gu.addClassToGraph(self.g, preferred_disease_id, dlabel)
# Add the disease to gene relationship.
rel_id = self._get_relationship_id(direct_evidence)
refs = self._process_pubmed_ids(pubmed_ids)
self._make_association(alt_locus, preferred_disease_id, rel_id, refs)
return
def _make_association(self, subject_id, object_id, rel_id, pubmed_ids):
"""
Make a reified association given an array of pubmed identifiers.
Args:
:param subject_id id of the subject of the association (gene/chem)
:param object_id id of the object of the association (disease)
:param rel_id relationship id
:param pubmed_ids an array of pubmed identifiers
Returns:
:return None
"""
# TODO pass in the relevant Assoc class rather than relying on G2P
assoc = G2PAssoc(self.name, subject_id, object_id, rel_id)
if pubmed_ids is not None and len(pubmed_ids) > 0:
eco = self._get_evidence_code('TAS')
for pmid in pubmed_ids:
r = Reference(pmid, Reference.ref_types['journal_article'])
r.addRefToGraph(self.g)
assoc.add_source(pmid)
assoc.add_evidence(eco)
assoc.add_association_to_graph(self.g)
assoc.load_all_properties(self.g)
return
@staticmethod
def _process_pubmed_ids(pubmed_ids):
"""
Take a list of pubmed IDs and add PMID prefix
Args:
:param pubmed_ids - string representing publication
ids seperated by a | symbol
Returns:
:return list: Pubmed curies
"""
if pubmed_ids.strip() == '':
id_list = []
else:
id_list = pubmed_ids.split('|')
for (i, val) in enumerate(id_list):
id_list[i] = 'PMID:'+val
return id_list
@staticmethod
def _get_evidence_code(evidence):
"""
Get curie for evidence class label
Args:
:param evidence (str): evidence label
Label:
:return str: curie for evidence label from ECO
"""
eco_map = {
'TAS': 'ECO:0000033'
}
return eco_map[evidence]
@staticmethod
def _get_relationship_id(rel):
"""
Get curie from relationship property label
Args:
:param rel (str): relationship label
Returns:
:return str: curie for relationship label
"""
gu = GraphUtils(curie_map.get())
rel_map = {
'therapeutic': gu.object_properties['substance_that_treats'],
'marker/mechanism': gu.object_properties['is_marker_for'],
}
return str(rel_map[rel])
@staticmethod
def _get_class_id(cls):
"""
Fet curie from CLASS_MAP dictionary
Args:
:param cls (str): class label
Returns:
:return str: curie for class label
"""
class_map = {
'pathway': 'PW:0000001',
'signal transduction': 'GO:0007165'
}
return class_map[cls]
def _parse_curated_chem_disease(self, limit):
line_counter = 0
file_path = '/'.join((self.rawdir, self.static_files['publications']['file']))
gu = GraphUtils(curie_map.get())
with open(file_path, 'r') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for row in reader:
# catch comment lines
if re.match('^#', ' '.join(row)):
continue
line_counter += 1
self._check_list_len(row, 10)
(pub_id, disease_label, disease_id, disease_cat, evidence,
chem_label, chem_id, cas_rn, gene_symbol, gene_acc) = row
rel_id = self._get_relationship_id(evidence)
chem_id = 'MESH:'+chem_id
gu.addClassToGraph(self.g, chem_id, chem_label)
gu.addClassToGraph(self.g, disease_id, None)
if pub_id != '':
pub_id = 'PMID:'+pub_id
r = Reference(pub_id, Reference.ref_types['journal_article'])
r.addRefToGraph(self.g)
else:
pub_id = None
self._make_association('MESH:'+chem_id, disease_id, rel_id, ['PMID:'+pub_id])
if not self.testMode and limit is not None and line_counter >= limit:
break
return
def getTestSuite(self):
import unittest
from tests.test_ctd import CTDTestCase
test_suite = unittest.TestLoader().loadTestsFromTestCase(CTDTestCase)
# test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(InteractionsTestCase))
return test_suite
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
line_counter = 0
fname = '/'.join((self.rawdir, self.files['catalog']['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = 'CL:0000034'
mouse_taxon = 'NCBITaxon:10090'
geno = Genotype(g)
with open(fname, 'r', encoding="utf8") as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
for row in filereader:
line_counter += 1
# skip the first 3 lines which are header, etc.
if line_counter < 4:
continue
(strain_id, strain_label, strain_type_symbol, strain_state,
mgi_allele_id, mgi_allele_symbol, mgi_allele_name,
mutation_type, chrom, mgi_gene_id, mgi_gene_symbol,
mgi_gene_name, sds_url, accepted_date, mp_ids, pubmed_nums,
research_areas) = row
if self.testMode and (strain_id not in self.test_ids) \
or mgi_gene_name == 'withdrawn':
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {
'variants': set(),
'genes': set()
}
# clean up the bad one
if mgi_allele_id == 'multiple mutation':
logger.error("Erroneous gene id: %s", mgi_allele_id)
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the
# sequence alteration types
# var_type =
# self._get_variant_type_from_abbrev(mutation_type)
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id,
# mgi_allele_id)
# scrub out any spaces
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id.strip() != '':
if re.match(r'Gene\s*ID:', mgi_gene_id, re.I):
mgi_gene_id = re.sub(r'Gene\s*ID:\s*', 'NCBIGene:',
mgi_gene_id)
elif not re.match(r'MGI', mgi_gene_id):
logger.info("Gene id not recognized: %s", mgi_gene_id)
if re.match(r'\d+$', mgi_gene_id):
# assume that if it's all numbers, then it's MGI
mgi_gene_id = 'MGI:' + str(mgi_gene_id)
logger.info("Assuming numerics are MGI.")
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors -
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol.strip() != '' and mgi_gene_id == '':
logger.error(
"Gene label with no identifier for strain %s: %s",
strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol.strip())
# make a temp id for genes that aren't identified
# tmp_gene_id = '_'+mgi_gene_symbol
# self.id_label_hash[tmp_gene_id] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mp_ids are now a comma delimited list
# with MP terms in brackets
phenotype_ids = []
if mp_ids != '':
for i in re.split(r',', mp_ids):
i = i.strip()
mps = re.search(r'\[(.*)\]', i)
if mps is not None:
mp_id = mps.group(1).strip()
phenotype_ids.append(mp_id)
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums.strip() != '':
for i in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:' + i.strip()
pubmed_ids.append(pmid)
r = Reference(g, pmid,
Reference.ref_types['journal_article'])
r.addRefToGraph()
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
model.addClassToGraph(mouse_taxon, None)
if research_areas.strip() == '':
research_areas = None
else:
research_areas = 'Research Areas: ' + research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
model.addIndividualToGraph(
strain_id, strain_label, strain_type,
research_areas) # an inst of mouse??
model.makeLeader(strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in the ontology
model.addClassToGraph(pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(
g, self.name, mgi_allele_id, pid,
model.object_properties['has_phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph()
else:
logger.info("Phenotypes and no allele for %s",
strain_id)
if not self.testMode and (limit is not None
and line_counter > limit):
break
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if len(variants) > 0:
for v in variants:
vl_id = v
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_:' + re.sub(r':', '', gene) + '-VL'
vl_symbol = self.id_label_hash[gene] + '<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = re.sub(r'^_', '', vl) + 'U'
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(
vslc_id, vl, None, geno.zygosity['indeterminate'],
geno.object_properties['has_alternate_part'], None)
model.addIndividualToGraph(
vslc_id, vslc_label,
geno.genoparts['variant_single_locus_complement'])
if len(vslc_list) > 0:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r'_|:', '', gvc_id)
gvc_id = '_:' + gvc_id
gvc_label = \
'; '.join(self.id_label_hash[v] for v in vslc_list)
model.addIndividualToGraph(
gvc_id, gvc_label,
geno.genoparts['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = \
re.sub(r':', '', '-'.join(
(geno.genoparts['unspecified_genomic_background'],
s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
bkgd_id = '_:' + bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified (' + s + ')',
geno.genoparts['unspecified_genomic_background'],
"A placeholder for the " +
"unspecified genetic background for " + s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
geno.genoparts['unspecified_genomic_background'])
geno.addParts(gvc_id, genotype_id,
geno.object_properties['has_alternate_part'])
geno.addGenotype(genotype_id, genotype_label)
g.addTriple(s, geno.object_properties['has_genotype'],
genotype_id)
else:
# logger.debug(
# "Strain %s is not making a proper genotype.", s)
pass
logger.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
return
def _process_data(self, raw, limit=None):
logger.info("Processing Data from %s", raw)
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
geno = Genotype(g)
line_counter = 0
impc_map = self.open_and_parse_yaml(self.map_files['impc_map'])
impress_map = json.loads(
self.fetch_from_url(
self.map_files['impress_map']).read().decode('utf-8'))
# Add the taxon as a class
taxon_id = 'NCBITaxon:10090' # map to Mus musculus
model.addClassToGraph(taxon_id, None)
# with open(raw, 'r', encoding="utf8") as csvfile:
with gzip.open(raw, 'rt') as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
next(filereader, None) # skip the header row
for row in filereader:
line_counter += 1
(marker_accession_id, marker_symbol, phenotyping_center,
colony, sex, zygosity, allele_accession_id, allele_symbol,
allele_name, strain_accession_id, strain_name, project_name,
project_fullname, pipeline_name, pipeline_stable_id,
procedure_stable_id, procedure_name, parameter_stable_id,
parameter_name, top_level_mp_term_id, top_level_mp_term_name,
mp_term_id, mp_term_name, p_value, percentage_change,
effect_size, statistical_method, resource_name) = row
if self.testMode and marker_accession_id not in self.test_ids:
continue
# ##### cleanup some of the identifiers ######
zygosity_id = self._map_zygosity(zygosity)
# colony ids sometimes have <> in them, spaces,
# or other non-alphanumerics and break our system;
# replace these with underscores
colony_id = '_:' + re.sub(r'\W+', '_', colony)
if not re.match(r'MGI', allele_accession_id):
allele_accession_id = \
'_:IMPC-'+re.sub(r':', '', allele_accession_id)
if re.search(r'EUROCURATE', strain_accession_id):
# the eurocurate links don't resolve at IMPC
strain_accession_id = '_:' + strain_accession_id
elif not re.match(r'MGI', strain_accession_id):
logger.info("Found a strange strain accession...%s",
strain_accession_id)
strain_accession_id = 'IMPC:' + strain_accession_id
######################
# first, add the marker and variant to the graph as with MGI,
# the allele is the variant locus. IF the marker is not known,
# we will call it a sequence alteration. otherwise,
# we will create a BNode for the sequence alteration.
sequence_alteration_id = variant_locus_id = None
variant_locus_name = sequence_alteration_name = None
# extract out what's within the <> to get the symbol
if re.match(r'.*<.*>', allele_symbol):
sequence_alteration_name = \
re.match(r'.*<(.*)>', allele_symbol).group(1)
else:
sequence_alteration_name = allele_symbol
if marker_accession_id is not None and \
marker_accession_id == '':
logger.warning("Marker unspecified on row %d",
line_counter)
marker_accession_id = None
if marker_accession_id is not None:
variant_locus_id = allele_accession_id
variant_locus_name = allele_symbol
variant_locus_type = geno.genoparts['variant_locus']
geno.addGene(marker_accession_id, marker_symbol,
geno.genoparts['gene'])
geno.addAllele(variant_locus_id, variant_locus_name,
variant_locus_type, None)
geno.addAlleleOfGene(variant_locus_id, marker_accession_id)
sequence_alteration_id = \
'_:seqalt'+re.sub(r':', '', allele_accession_id)
geno.addSequenceAlterationToVariantLocus(
sequence_alteration_id, variant_locus_id)
else:
sequence_alteration_id = allele_accession_id
# IMPC contains targeted mutations with either gene traps,
# knockouts, insertion/intragenic deletions.
# but I don't really know what the SeqAlt is here,
# so I don't add it.
geno.addSequenceAlteration(sequence_alteration_id,
sequence_alteration_name)
# ############# BUILD THE COLONY #############
# First, let's describe the colony that the animals come from
# The Colony ID refers to the ES cell clone
# used to generate a mouse strain.
# Terry sez: we use this clone ID to track
# ES cell -> mouse strain -> mouse phenotyping.
# The same ES clone maybe used at multiple centers,
# so we have to concatenate the two to have a unique ID.
# some useful reading about generating mice from ES cells:
# http://ki.mit.edu/sbc/escell/services/details
# here, we'll make a genotype
# that derives from an ES cell with a given allele.
# the strain is not really attached to the colony.
# the colony/clone is reflective of the allele,
# with unknown zygosity
stem_cell_class = 'ERO:0002002'
model.addIndividualToGraph(colony_id, colony, stem_cell_class)
# vslc of the colony has unknown zygosity
# note that we will define the allele
# (and it's relationship to the marker, etc.) later
# FIXME is it really necessary to create this vslc
# when we always know it's unknown zygosity?
vslc_colony = \
'_:'+re.sub(r':', '', allele_accession_id+geno.zygosity['indeterminate'])
vslc_colony_label = allele_symbol + '/<?>'
# for ease of reading, we make the colony genotype variables.
# in the future, it might be desired to keep the vslcs
colony_genotype_id = vslc_colony
colony_genotype_label = vslc_colony_label
geno.addGenotype(colony_genotype_id, colony_genotype_label)
geno.addParts(allele_accession_id, colony_genotype_id,
geno.object_properties['has_alternate_part'])
geno.addPartsToVSLC(
vslc_colony, allele_accession_id, None,
geno.zygosity['indeterminate'],
geno.object_properties['has_alternate_part'])
g.addTriple(colony_id, geno.object_properties['has_genotype'],
colony_genotype_id)
# ########## BUILD THE ANNOTATED GENOTYPE ##########
# now, we'll build the genotype of the individual that derives
# from the colony/clone genotype that is attached to
# phenotype = colony_id + strain + zygosity + sex
# (and is derived from a colony)
# this is a sex-agnostic genotype
genotype_id = \
self.make_id(
(colony_id + phenotyping_center + zygosity +
strain_accession_id))
geno.addSequenceDerivesFrom(genotype_id, colony_id)
# build the VSLC of the sex-agnostic genotype
# based on the zygosity
allele1_id = allele_accession_id
allele2_id = allele2_rel = None
allele1_label = allele_symbol
allele2_label = '<?>'
# Making VSLC labels from the various parts,
# can change later if desired.
if zygosity == 'heterozygote':
allele2_label = re.sub(r'<.*', '<+>', allele1_label)
allele2_id = None
elif zygosity == 'homozygote':
allele2_label = allele1_label
allele2_id = allele1_id
allele2_rel = geno.object_properties['has_alternate_part']
elif zygosity == 'hemizygote':
allele2_label = re.sub(r'<.*', '<0>', allele1_label)
allele2_id = None
elif zygosity == 'not_applicable':
allele2_label = re.sub(r'<.*', '<?>', allele1_label)
allele2_id = None
else:
logger.warning("found unknown zygosity %s", zygosity)
break
vslc_name = '/'.join((allele1_label, allele2_label))
# Add the VSLC
vslc_id = '-'.join(
(marker_accession_id, allele_accession_id, zygosity))
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
model.addIndividualToGraph(
vslc_id, vslc_name,
geno.genoparts['variant_single_locus_complement'])
geno.addPartsToVSLC(
vslc_id, allele1_id, allele2_id, zygosity_id,
geno.object_properties['has_alternate_part'], allele2_rel)
# add vslc to genotype
geno.addVSLCtoParent(vslc_id, genotype_id)
# note that the vslc is also the gvc
model.addType(
vslc_id,
Genotype.genoparts['genomic_variation_complement'])
# Add the genomic background
# create the genomic background id and name
if strain_accession_id != '':
genomic_background_id = strain_accession_id
else:
genomic_background_id = None
genotype_name = vslc_name
if genomic_background_id is not None:
geno.addGenotype(genomic_background_id, strain_name,
geno.genoparts['genomic_background'])
# make a phenotyping-center-specific strain
# to use as the background
pheno_center_strain_label = \
strain_name + '-' + phenotyping_center + '-' + colony
pheno_center_strain_id = \
'-'.join((re.sub(r':', '', genomic_background_id),
re.sub(r'\s', '_', phenotyping_center),
re.sub(r'\W+', '', colony)))
if not re.match(r'^_', pheno_center_strain_id):
pheno_center_strain_id = '_:' + pheno_center_strain_id
geno.addGenotype(pheno_center_strain_id,
pheno_center_strain_label,
geno.genoparts['genomic_background'])
geno.addSequenceDerivesFrom(pheno_center_strain_id,
genomic_background_id)
# Making genotype labels from the various parts,
# can change later if desired.
# since the genotype is reflective of the place
# it got made, should put that in to disambiguate
genotype_name = \
genotype_name+' ['+pheno_center_strain_label+']'
geno.addGenomicBackgroundToGenotype(
pheno_center_strain_id, genotype_id)
geno.addTaxon(taxon_id, pheno_center_strain_id)
# this is redundant, but i'll keep in in for now
geno.addSequenceDerivesFrom(genotype_id, colony_id)
geno.addGenotype(genotype_id, genotype_name)
# Make the sex-qualified genotype,
# which is what the phenotype is associated with
sex_qualified_genotype_id = \
self.make_id(
(colony_id + phenotyping_center + zygosity +
strain_accession_id+sex))
sex_qualified_genotype_label = genotype_name + ' (' + sex + ')'
if sex == 'male':
sq_type_id = geno.genoparts['male_genotype']
elif sex == 'female':
sq_type_id = geno.genoparts['female_genotype']
else:
sq_type_id = geno.genoparts['sex_qualified_genotype']
geno.addGenotype(sex_qualified_genotype_id,
sex_qualified_genotype_label, sq_type_id)
geno.addParts(genotype_id, sex_qualified_genotype_id,
geno.object_properties['has_alternate_part'])
if genomic_background_id is not None and \
genomic_background_id != '':
# Add the taxon to the genomic_background_id
geno.addTaxon(taxon_id, genomic_background_id)
else:
# add it as the genomic background
geno.addTaxon(taxon_id, genotype_id)
# ############# BUILD THE G2P ASSOC #############
# from an old email dated July 23 2014:
# Phenotypes associations are made to
# imits colony_id+center+zygosity+gender
phenotype_id = mp_term_id
# it seems that sometimes phenotype ids are missing.
# indicate here
if phenotype_id is None or phenotype_id == '':
logger.warning("No phenotype id specified for row %d: %s",
line_counter, str(row))
continue
# hard coded ECO code
eco_id = "ECO:0000015"
# the association comes as a result of a g2p from
# a procedure in a pipeline at a center and parameter tested
assoc = G2PAssoc(g, self.name, sex_qualified_genotype_id,
phenotype_id)
assoc.add_evidence(eco_id)
# assoc.set_score(float(p_value))
# TODO add evidence instance using
# pipeline_stable_id +
# procedure_stable_id +
# parameter_stable_id
assoc.add_association_to_graph()
assoc_id = assoc.get_association_id()
# add a free-text description
try:
description = \
' '.join((mp_term_name, 'phenotype determined by',
phenotyping_center, 'in an',
procedure_name, 'assay where',
parameter_name.strip(),
'was measured with an effect_size of',
str(round(float(effect_size), 5)),
'(p =', "{:.4e}".format(float(p_value)), ').'))
except ValueError:
description = \
' '.join((mp_term_name, 'phenotype determined by',
phenotyping_center, 'in an',
procedure_name, 'assay where',
parameter_name.strip(),
'was measured with an effect_size of',
str(effect_size),
'(p =', "{0}".format(p_value), ').'))
study_bnode = \
self._add_study_provenance(
impc_map, impress_map, phenotyping_center, colony,
project_fullname, pipeline_name, pipeline_stable_id,
procedure_stable_id, procedure_name,
parameter_stable_id, parameter_name,
statistical_method, resource_name)
evidence_line_bnode = \
self._add_evidence(
assoc_id, eco_id, impc_map, p_value, percentage_change,
effect_size, study_bnode)
self._add_assertion_provenance(assoc_id, evidence_line_bnode,
impc_map)
model.addDescription(evidence_line_bnode, description)
# resource_id = resource_name
# assoc.addSource(g, assoc_id, resource_id)
if not self.testMode and \
limit is not None and line_counter > limit:
break
return
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
gu = GraphUtils(curie_map.get())
fname = '/'.join((self.rawdir, self.files['catalog']['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = 'CL:0000034'
mouse_taxon = 'NCBITaxon:10090'
geno = Genotype(g)
with open(fname, 'r', encoding="utf8") as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
for row in filereader:
line_counter += 1
# skip the first 3 lines which are header, etc.
if line_counter < 4:
continue
(strain_id, strain_label, strain_type_symbol, strain_state,
mgi_allele_id, mgi_allele_symbol, mgi_allele_name,
mutation_type, chrom, mgi_gene_id, mgi_gene_symbol,
mgi_gene_name, sds_url, accepted_date, mp_ids, pubmed_nums,
research_areas) = row
if self.testMode and (strain_id not in self.test_ids):
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {'variants': set(),
'genes': set()}
# clean up the bad one
if mgi_allele_id == 'multiple mutation':
logger.error("Erroneous gene id: %s", mgi_allele_id)
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the
# sequence alteration types
# var_type =
# self._get_variant_type_from_abbrev(mutation_type)
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id,
# mgi_allele_id)
# scrub out any spaces
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id.strip() != '':
if re.match(r'Gene\s*ID:', mgi_gene_id, re.I):
mgi_gene_id = re.sub(r'Gene\s*ID:\s*', 'NCBIGene:',
mgi_gene_id)
elif not re.match(r'MGI', mgi_gene_id):
logger.info("Gene id not recognized: %s", mgi_gene_id)
if re.match(r'\d+$', mgi_gene_id):
# assume that if it's all numbers, then it's MGI
mgi_gene_id = 'MGI:'+str(mgi_gene_id)
logger.info("Assuming numerics are MGI.")
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors -
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol.strip() != '' and mgi_gene_id == '':
logger.error(
"Gene label with no identifier for strain %s: %s",
strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol.strip())
# make a temp id for genes that aren't identified
# tmp_gene_id = '_'+mgi_gene_symbol
# self.id_label_hash[tmp_gene_id] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mp_ids are now a comma delimited list
# with MP terms in brackets
phenotype_ids = []
if mp_ids != '':
for i in re.split(r',', mp_ids):
i = i.strip()
mps = re.search(r'\[(.*)\]', i)
if mps is not None:
mp_id = mps.group(1).strip()
phenotype_ids.append(mp_id)
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums.strip() != '':
for i in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:'+i.strip()
pubmed_ids.append(pmid)
r = Reference(pmid,
Reference.ref_types['journal_article'])
r.addRefToGraph(g)
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
gu.addClassToGraph(g, mouse_taxon, None)
if research_areas.strip() == '':
research_areas = None
else:
research_areas = 'Research Areas: '+research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
gu.addIndividualToGraph(
g, strain_id, strain_label, strain_type,
research_areas) # an inst of mouse??
gu.makeLeader(g, strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in the ontology
gu.addClassToGraph(g, pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(self.name, mgi_allele_id, pid,
gu.object_properties['has_phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph(g)
else:
logger.info("Phenotypes and no allele for %s",
strain_id)
if not self.testMode and (
limit is not None and line_counter > limit):
break
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if len(variants) > 0:
for v in variants:
vl_id = v
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_'+gene+'-VL'
vl_id = re.sub(r':', '', vl_id)
if self.nobnodes:
vl_id = ':'+vl_id
vl_symbol = self.id_label_hash[gene]+'<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = '_'+re.sub(r'^_', '', vl)+'U'
vslc_id = re.sub(r':', '', vslc_id)
if self.nobnodes:
vslc_id = ':' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(
vslc_id, vl, None, geno.zygosity['indeterminate'],
geno.object_properties['has_alternate_part'], None)
gu.addIndividualToGraph(
g, vslc_id, vslc_label,
geno.genoparts['variant_single_locus_complement'])
if len(vslc_list) > 0:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r':', '', gvc_id)
if self.nobnodes:
gvc_id = ':'+gvc_id
gvc_label = \
'; '.join(self.id_label_hash[v] for v in vslc_list)
gu.addIndividualToGraph(
g, gvc_id, gvc_label,
geno.genoparts['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = \
'_' + re.sub(r':', '', '-'.join(
(geno.genoparts['unspecified_genomic_background'],
s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
if self.nobnodes:
bkgd_id = ':'+bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified ('+s+')',
geno.genoparts['unspecified_genomic_background'],
"A placeholder for the " +
"unspecified genetic background for "+s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
geno.genoparts['unspecified_genomic_background'])
geno.addParts(
gvc_id, genotype_id,
geno.object_properties['has_alternate_part'])
geno.addGenotype(genotype_id, genotype_label)
gu.addTriple(
g, s, geno.object_properties['has_genotype'],
genotype_id)
else:
# logger.debug(
# "Strain %s is not making a proper genotype.", s)
pass
gu.loadProperties(
g, G2PAssoc.object_properties, G2PAssoc.OBJECTPROP)
gu.loadProperties(
g, G2PAssoc.datatype_properties, G2PAssoc.DATAPROP)
gu.loadProperties(
g, G2PAssoc.annotation_properties, G2PAssoc.ANNOTPROP)
gu.loadAllProperties(g)
logger.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
return
def process_catalog(self, limit=None):
"""
:param limit:
:return:
"""
raw = '/'.join((self.rawdir, self.files['catalog']['file']))
logger.info("Processing Data from %s", raw)
gu = GraphUtils(curie_map.get())
if self.testMode: # set the graph to build
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu.loadProperties(g, geno.object_properties, gu.OBJPROP)
gu.loadAllProperties(g)
tax_id = 'NCBITaxon:9606' # hardcode
genome_version = 'GRCh38' # hardcode
# build a hashmap of genomic location to identifiers,
# to try to get the equivalences
loc_to_id_hash = {}
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
next(filereader, None) # skip the header row
for row in filereader:
if not row:
pass
else:
line_counter += 1
(date_added_to_catalog, pubmed_num, first_author, pub_date,
journal, link, study_name, disease_or_trait,
initial_sample_description, replicate_sample_description,
region, chrom_num, chrom_pos, reported_gene_nums,
mapped_gene, upstream_gene_num, downstream_gene_num,
snp_gene_nums, upstream_gene_distance,
downstream_gene_distance, strongest_snp_risk_allele, snps,
merged, snp_id_current, context, intergenic_flag,
risk_allele_frequency, pvalue, pvalue_mlog, pvalue_text,
or_or_beta, confidence_interval_95,
platform_with_snps_passing_qc, cnv_flag, mapped_trait,
mapped_trait_uri) = row
intersect = \
list(set([str(i) for i in self.test_ids['gene']]) &
set(re.split(r',', snp_gene_nums)))
# skip if no matches found in test set
if self.testMode and len(intersect) == 0:
continue
# 06-May-2015 25917933 Zai CC 20-Nov-2014 J Psychiatr Res http://europepmc.org/abstract/MED/25917933
# A genome-wide association study of suicide severity scores in bipolar disorder.
# Suicide in bipolar disorder
# 959 European ancestry individuals NA
# 10p11.22 10 32704340 C10orf68, CCDC7, ITGB1 CCDC7
# rs7079041-A rs7079041 0 7079041 intron 0 2E-6 5.698970
if chrom_num != '' and chrom_pos != '':
loc = 'chr'+str(chrom_num)+':'+str(chrom_pos)
if loc not in loc_to_id_hash:
loc_to_id_hash[loc] = set()
else:
loc = None
if re.search(r' x ', strongest_snp_risk_allele) \
or re.search(r',', strongest_snp_risk_allele):
# TODO deal with haplotypes
logger.warning(
"We can't deal with haplotypes yet: %s",
strongest_snp_risk_allele)
continue
elif re.match(r'rs', strongest_snp_risk_allele):
rs_id = 'dbSNP:'+strongest_snp_risk_allele.strip()
# remove the alteration
elif re.match(r'kgp', strongest_snp_risk_allele):
# FIXME this isn't correct
rs_id = 'dbSNP:'+strongest_snp_risk_allele.strip()
# http://www.1000genomes.org/faq/what-are-kgp-identifiers
# for some information
# They were created by Illumina for their genotyping
# platform before some variants identified during the
# pilot phase of the project had been assigned
# rs numbers.
elif re.match(r'chr', strongest_snp_risk_allele):
# like: chr10:106180121-G
rs_id = ':gwas-' + \
re.sub(
r':', '-', strongest_snp_risk_allele.strip())
elif strongest_snp_risk_allele.strip() == '':
# logger.debug(
# "No strongest SNP risk allele for %s:\n%s",
# pubmed_num, str(row))
# FIXME still consider adding in the EFO terms
# for what the study measured?
continue
else:
logger.warning(
"There's a snp id i can't manage: %s",
strongest_snp_risk_allele)
continue
alteration = re.search(r'-(.*)$', rs_id)
if alteration is not None \
and re.match(r'[ATGC]', alteration.group(1)):
# add variation to snp
pass # TODO
rs_id = re.sub(r'-.*$', '', rs_id).strip()
if loc is not None:
loc_to_id_hash[loc].add(rs_id)
pubmed_id = 'PMID:'+pubmed_num
r = Reference(
pubmed_id, Reference.ref_types['journal_article'])
r.addRefToGraph(g)
# create the chromosome
chrom_id = makeChromID(chrom_num, genome_version, 'CHR')
# add the feature to the graph
snp_description = None
if risk_allele_frequency != '' and \
risk_allele_frequency != 'NR':
snp_description = \
str(risk_allele_frequency) + \
' [risk allele frequency]'
f = Feature(
rs_id, strongest_snp_risk_allele.strip(),
Feature.types[r'SNP'], snp_description)
if chrom_num != '' and chrom_pos != '':
f.addFeatureStartLocation(chrom_pos, chrom_id)
f.addFeatureEndLocation(chrom_pos, chrom_id)
f.addFeatureToGraph(g)
f.addTaxonToFeature(g, tax_id)
# TODO consider adding allele frequency as property;
# but would need background info to do that
# also want to add other descriptive info about
# the variant from the context
for c in re.split(r';', context):
cid = self._map_variant_type(c.strip())
if cid is not None:
gu.addType(g, rs_id, cid)
# add deprecation information
if merged == 1 and str(snp_id_current.strip()) != '':
# get the current rs_id
current_rs_id = 'dbSNP:'
if not re.match(r'rs', snp_id_current):
current_rs_id += 'rs'
if loc is not None:
loc_to_id_hash[loc].append(current_rs_id)
current_rs_id += str(snp_id_current)
gu.addDeprecatedIndividual(g, rs_id, current_rs_id)
# TODO check on this
# should we add the annotations to the current
# or orig?
gu.makeLeader(g, current_rs_id)
else:
gu.makeLeader(g, rs_id)
# add the feature as a sequence alteration
# affecting various genes
# note that intronic variations don't necessarily list
# the genes such as for rs10448080 FIXME
if snp_gene_nums != '':
for s in re.split(r',', snp_gene_nums):
s = s.strip()
# still have to test for this,
# because sometimes there's a leading comma
if s != '':
gene_id = 'NCBIGene:'+s
geno.addAlleleOfGene(rs_id, gene_id)
# add the up and downstream genes if they are available
if upstream_gene_num != '':
downstream_gene_id = 'NCBIGene:'+downstream_gene_num
gu.addTriple(
g, rs_id,
Feature.object_properties[
r'upstream_of_sequence_of'],
downstream_gene_id)
if downstream_gene_num != '':
upstream_gene_id = 'NCBIGene:'+upstream_gene_num
gu.addTriple(
g, rs_id,
Feature.object_properties[
'downstream_of_sequence_of'],
upstream_gene_id)
description = 'A study of ' + disease_or_trait + \
' in ' + initial_sample_description
if replicate_sample_description != '':
description = \
' '.join(
(description, 'with',
replicate_sample_description))
if platform_with_snps_passing_qc != '':
description = ' '.join(
(description, 'on platform',
platform_with_snps_passing_qc))
description = ' '.join((description, '(p='+pvalue+')'))
# make associations to the EFO terms; there can be >1
if mapped_trait_uri.strip() != '':
for t in re.split(r',', mapped_trait_uri):
t = t.strip()
cu = CurieUtil(curie_map.get())
tid = cu.get_curie(t)
assoc = G2PAssoc(
self.name, rs_id, tid,
gu.object_properties['contributes_to'])
assoc.add_source(pubmed_id)
# combinatorial evidence
# used in automatic assertion
eco_id = 'ECO:0000213'
assoc.add_evidence(eco_id)
# assoc.set_description(description)
# FIXME score should get added to provenance/study
# assoc.set_score(pvalue)
assoc.add_association_to_graph(g)
if not self.testMode and\
(limit is not None and line_counter > limit):
break
Assoc(self.name).load_all_properties(g)
# loop through the location hash,
# and make all snps at that location equivalent
for l in loc_to_id_hash:
snp_ids = loc_to_id_hash[l]
if len(snp_ids) > 1:
logger.info("%s has >1 snp id: %s", l, str(snp_ids))
return
def _get_variants(self, limit):
"""
Currently loops through the variant_summary file.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
geno = Genotype(g)
f = Feature(g, None, None, None)
# add the taxon and the genome
tax_num = '9606' # HARDCODE
tax_id = 'NCBITaxon:' + tax_num
tax_label = 'Human'
model.addClassToGraph(tax_id, None)
geno.addGenome(tax_id, tax_label) # label gets added elsewhere
# not unzipping the file
logger.info("Processing Variant records")
line_counter = 0
myfile = '/'.join((self.rawdir, self.files['variant_summary']['file']))
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match(r'^#', line):
continue
# AlleleID integer value as stored in the AlleleID field in ClinVar (//Measure/@ID in the XML)
# Type character, the type of variation
# Name character, the preferred name for the variation
# GeneID integer, GeneID in NCBI's Gene database
# GeneSymbol character, comma-separated list of GeneIDs overlapping the variation
# ClinicalSignificance character, comma-separated list of values of clinical significance reported for this variation
# for the mapping between the terms listed here and the integers in the .VCF files, see
# http://www.ncbi.nlm.nih.gov/clinvar/docs/clinsig/
# RS# (dbSNP) integer, rs# in dbSNP
# nsv (dbVar) character, the NSV identifier for the region in dbVar
# RCVaccession character, list of RCV accessions that report this variant
# TestedInGTR character, Y/N for Yes/No if there is a test registered as specific to this variation in the NIH Genetic Testing Registry (GTR)
# PhenotypeIDs character, list of db names and identifiers for phenotype(s) reported for this variant
# Origin character, list of all allelic origins for this variation
# Assembly character, name of the assembly on which locations are based
# Chromosome character, chromosomal location
# Start integer, starting location, in pter->qter orientation
# Stop integer, end location, in pter->qter orientation
# Cytogenetic character, ISCN band
# ReviewStatus character, highest review status for reporting this measure. For the key to the terms,
# and their relationship to the star graphics ClinVar displays on its web pages,
# see http://www.ncbi.nlm.nih.gov/clinvar/docs/variation_report/#interpretation
# HGVS(c.) character, RefSeq cDNA-based HGVS expression
# HGVS(p.) character, RefSeq protein-based HGVS expression
# NumberSubmitters integer, number of submissions with this variant
# LastEvaluated datetime, the latest time any submitter reported clinical significance
# Guidelines character, ACMG only right now, for the reporting of incidental variation in a Gene
# (NOTE: if ACMG, not a specific to the allele but to the Gene)
# OtherIDs character, list of other identifiers or sources of information about this variant
# VariantID integer, the value used to build the URL for the current default report,
# e.g. http://www.ncbi.nlm.nih.gov/clinvar/variation/1756/
#
# a crude check that there's an expected number of cols.
# if not, error out because something changed.
num_cols = len(line.split('\t'))
expected_numcols = 29
if num_cols != expected_numcols:
logger.error(
"Unexpected number of columns in raw file " +
"(%d actual vs %d expected)", num_cols,
expected_numcols)
(allele_num, allele_type, allele_name, gene_num, gene_symbol,
clinical_significance, dbsnp_num, dbvar_num, rcv_nums,
tested_in_gtr, phenotype_ids, origin, assembly, chr, start,
stop, cytogenetic_loc, review_status, hgvs_c, hgvs_p,
number_of_submitters, last_eval, guidelines, other_ids,
variant_num, reference_allele, alternate_allele, categories,
ChromosomeAccession) = line.split('\t')
# ###set filter=None in init if you don't want to have a filter
# if self.filter is not None:
# if ((self.filter == 'taxids' and\
# (int(tax_num) not in self.tax_ids)) or\
# (self.filter == 'geneids' and\
# (int(gene_num) not in self.gene_ids))):
# continue
# #### end filter
line_counter += 1
pheno_list = []
if phenotype_ids != '-':
# trim any leading/trailing semicolons/commas
phenotype_ids = re.sub(r'^[;,]', '', phenotype_ids)
phenotype_ids = re.sub(r'[;,]$', '', phenotype_ids)
pheno_list = re.split(r'[,;]', phenotype_ids)
if self.testMode:
# get intersection of test disease ids
# and these phenotype_ids
intersect = \
list(
set([str(i)
for i in self.disease_ids]) & set(pheno_list))
if int(gene_num) not in self.gene_ids and\
int(variant_num) not in self.variant_ids and\
len(intersect) < 1:
continue
# TODO may need to switch on assembly to create correct
# assembly/build identifiers
build_id = ':'.join(('NCBIGenome', assembly))
# make the reference genome build
geno.addReferenceGenome(build_id, assembly, tax_id)
allele_type_id = self._map_type_of_allele(allele_type)
bandinbuild_id = None
if str(chr) == '':
# check cytogenic location
if str(cytogenetic_loc).strip() != '':
# use cytogenic location to get the apx location
# oddly, they still put an assembly number even when
# there's no numeric location
if not re.search(r'-', str(cytogenetic_loc)):
band_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)), tax_num,
'CHR')
geno.addChromosomeInstance(cytogenetic_loc,
build_id, assembly,
band_id)
bandinbuild_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)), assembly,
'MONARCH')
else:
# can't deal with ranges yet
pass
else:
# add the human chromosome class to the graph,
# and add the build-specific version of it
chr_id = makeChromID(str(chr), tax_num, 'CHR')
geno.addChromosomeClass(str(chr), tax_id, tax_label)
geno.addChromosomeInstance(str(chr), build_id, assembly,
chr_id)
chrinbuild_id = makeChromID(str(chr), assembly, 'MONARCH')
seqalt_id = ':'.join(('ClinVarVariant', variant_num))
gene_id = None
# they use -1 to indicate unknown gene
if str(gene_num) != '-1' and str(gene_num) != 'more than 10':
if re.match(r'^Gene:', gene_num):
gene_num = "NCBI" + gene_num
else:
gene_id = ':'.join(('NCBIGene', str(gene_num)))
# FIXME there are some "variants" that are actually haplotypes
# probably will get taken care of when we switch to processing
# the xml for example, variant_num = 38562
# but there's no way to tell if it's a haplotype
# in the csv data so the dbsnp or dbvar
# should probably be primary,
# and the variant num be the vslc,
# with each of the dbsnps being added to it
# TODO clinical significance needs to be mapped to
# a list of terms
# first, make the variant:
f = Feature(seqalt_id, allele_name, allele_type_id)
if start != '-' and start.strip() != '':
f.addFeatureStartLocation(start, chrinbuild_id)
if stop != '-' and stop.strip() != '':
f.addFeatureEndLocation(stop, chrinbuild_id)
f.addFeatureToGraph()
f.addTaxonToFeature(tax_id)
# make the ClinVarVariant the clique leader
model.makeLeader(seqalt_id)
if bandinbuild_id is not None:
f.addSubsequenceOfFeature(bandinbuild_id)
# CHECK - this makes the assumption that there is
# only one affected chromosome per variant what happens with
# chromosomal rearrangement variants?
# shouldn't both chromosomes be here?
# add the hgvs as synonyms
if hgvs_c != '-' and hgvs_c.strip() != '':
model.addSynonym(seqalt_id, hgvs_c)
if hgvs_p != '-' and hgvs_p.strip() != '':
model.addSynonym(seqalt_id, hgvs_p)
# add the dbsnp and dbvar ids as equivalent
if dbsnp_num != '-' and int(dbsnp_num) != -1:
dbsnp_id = 'dbSNP:rs' + str(dbsnp_num)
model.addIndividualToGraph(dbsnp_id, None)
model.addSameIndividual(seqalt_id, dbsnp_id)
if dbvar_num != '-':
dbvar_id = 'dbVar:' + dbvar_num
model.addIndividualToGraph(dbvar_id, None)
model.addSameIndividual(seqalt_id, dbvar_id)
# TODO - not sure if this is right... add as xref?
# the rcv is like the combo of the phenotype with the variant
if rcv_nums != '-':
for rcv_num in re.split(r';', rcv_nums):
rcv_id = 'ClinVar:' + rcv_num
model.addIndividualToGraph(rcv_id, None)
model.addXref(seqalt_id, rcv_id)
if gene_id is not None:
# add the gene
model.addClassToGraph(gene_id, gene_symbol)
# make a variant locus
vl_id = '_' + gene_num + '-' + variant_num
if self.nobnodes:
vl_id = ':' + vl_id
vl_label = allele_name
model.addIndividualToGraph(vl_id, vl_label,
geno.genoparts['variant_locus'])
geno.addSequenceAlterationToVariantLocus(seqalt_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
# some basic reporting
gmatch = re.search(r'\(\w+\)', allele_name)
if gmatch is not None and len(gmatch.groups()) > 0:
logger.info(
"Gene found in allele label, but no id provided: %s",
gmatch.group(1))
elif re.match(r'more than 10', gene_symbol):
logger.info(
"More than 10 genes found; "
"need to process XML to fetch (variant=%d)",
int(variant_num))
else:
logger.info("No gene listed for variant %d",
int(variant_num))
# parse the list of "phenotypes" which are diseases.
# add them as an association
# ;GeneReviews:NBK1440,MedGen:C0392514,OMIM:235200,SNOMED CT:35400008;MedGen:C3280096,OMIM:614193;MedGen:CN034317,OMIM:612635;MedGen:CN169374
# the list is both semicolon delimited and comma delimited,
# but i don't know why! some are bad, like:
# Orphanet:ORPHA ORPHA319705,SNOMED CT:49049000
if phenotype_ids != '-':
for phenotype in pheno_list:
m = re.match(r"(Orphanet:ORPHA(?:\s*ORPHA)?)",
phenotype)
if m is not None and len(m.groups()) > 0:
phenotype = re.sub(m.group(1), 'Orphanet:',
phenotype.strip())
elif re.match(r'ORPHA:\d+', phenotype):
phenotype = re.sub(r'^ORPHA', 'Orphanet',
phenotype.strip())
elif re.match(r'Human Phenotype Ontology', phenotype):
phenotype = re.sub(r'^Human Phenotype Ontology',
'', phenotype.strip())
elif re.match(r'SNOMED CT:\s?', phenotype):
phenotype = re.sub(r'SNOMED CT:\s?', 'SNOMED:',
phenotype.strip())
elif re.match(r'^Gene:', phenotype):
continue
assoc = G2PAssoc(g, self.name, seqalt_id,
phenotype.strip())
assoc.add_association_to_graph()
if other_ids != '-':
id_list = other_ids.split(',')
# process the "other ids" ex:
# CFTR2:F508del,HGMD:CD890142,OMIM Allelic Variant:602421.0001
# TODO make more xrefs
for xrefid in id_list:
prefix = xrefid.split(':')[0].strip()
if prefix == 'OMIM Allelic Variant':
xrefid = 'OMIM:' + xrefid.split(':')[1]
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'HGMD':
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'dbVar' \
and dbvar_num == xrefid.split(':')[1].strip():
pass # skip over this one
elif re.search(r'\s', prefix):
pass
# logger.debug(
# 'xref prefix has a space: %s', xrefid)
else:
# should be a good clean prefix
# note that HGMD variants are in here as Xrefs
# because we can't resolve URIs for them
# logger.info("Adding xref: %s", xrefid)
# gu.addXref(g, seqalt_id, xrefid)
# logger.info("xref prefix to add: %s", xrefid)
pass
if not self.testMode and limit is not None \
and line_counter > limit:
break
logger.info("Finished parsing variants")
return
def _process_diseasegene(self, limit):
"""
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
myfile = '/'.join((self.rawdir, self.files['disease-gene']['file']))
# PYLINT complains iterparse deprecated,
# but as of py 3.4 only the optional & unsupplied parse arg is.
for event, elem in ET.iterparse(myfile):
if elem.tag == 'Disorder':
# get the element name and id, ignoreS element name
# id = elem.get('id') # some internal identifier
disorder_num = elem.find('OrphaNumber').text
disorder_id = 'Orphanet:'+str(disorder_num)
if self.testMode and \
disorder_id not in \
config.get_config()['test_ids']['disease']:
continue
disorder_label = elem.find('Name').text
# make a hash of internal gene id to type for later lookup
gene_iid_to_type = {}
gene_list = elem.find('GeneList')
for gene in gene_list.findall('Gene'):
gene_iid = gene.get('id')
gene_type = gene.find('GeneType').get('id')
gene_iid_to_type[gene_iid] = gene_type
# assuming that these are in the ontology
gu.addClassToGraph(g, disorder_id, disorder_label)
assoc_list = elem.find('DisorderGeneAssociationList')
for a in assoc_list.findall('DisorderGeneAssociation'):
gene_iid = a.find('.//Gene').get('id')
gene_name = a.find('.//Gene/Name').text
gene_symbol = a.find('.//Gene/Symbol').text
gene_num = a.find('./Gene/OrphaNumber').text
gene_id = 'Orphanet:'+str(gene_num)
gene_type_id = \
self._map_gene_type_id(gene_iid_to_type[gene_iid])
gu.addClassToGraph(
g, gene_id, gene_symbol, gene_type_id, gene_name)
syn_list = a.find('./Gene/SynonymList')
if int(syn_list.get('count')) > 0:
for s in syn_list.findall('./Synonym'):
gu.addSynonym(g, gene_id, s.text)
dgtype = a.find('DisorderGeneAssociationType').get('id')
rel_id = self._map_rel_id(dgtype)
dg_label = \
a.find('./DisorderGeneAssociationType/Name').text
if rel_id is None:
logger.warning(
"Cannot map association type (%s) to RO " +
"for association (%s | %s). Skipping.",
dg_label, disorder_label, gene_symbol)
continue
alt_locus_id = '_'+gene_num+'-'+disorder_num+'VL'
alt_label = \
' '.join(('some variant of', gene_symbol.strip(),
'that is a', dg_label.lower(),
disorder_label))
if self.nobnodes:
alt_locus_id = ':'+alt_locus_id
gu.addIndividualToGraph(g, alt_locus_id, alt_label,
geno.genoparts['variant_locus'])
geno.addAlleleOfGene(alt_locus_id, gene_id)
# consider typing the gain/loss-of-function variants like:
# http://sequenceontology.org/browser/current_svn/term/SO:0002054
# http://sequenceontology.org/browser/current_svn/term/SO:0002053
# use "assessed" status to issue an evidence code
# FIXME I think that these codes are sub-optimal
status_code = \
a.find('DisorderGeneAssociationStatus').get('id')
# imported automatically asserted information
# used in automatic assertion
eco_id = 'ECO:0000323'
# Assessed
# TODO are these internal ids stable between releases?
if status_code == '17991':
# imported manually asserted information
# used in automatic assertion
eco_id = 'ECO:0000322'
# Non-traceable author statement ECO_0000034
# imported information in automatic assertion ECO_0000313
assoc = G2PAssoc(self.name, alt_locus_id,
disorder_id, rel_id)
assoc.add_evidence(eco_id)
assoc.add_association_to_graph(g)
rlist = a.find('./Gene/ExternalReferenceList')
eqid = None
for r in rlist.findall('ExternalReference'):
if r.find('Source').text == 'Ensembl':
eqid = 'ENSEMBL:'+r.find('Reference').text
elif r.find('Source').text == 'HGNC':
eqid = 'HGNC:'+r.find('Reference').text
elif r.find('Source').text == 'OMIM':
eqid = 'OMIM:'+r.find('Reference').text
else:
pass # skip the others for now
if eqid is not None:
gu.addClassToGraph(g, eqid, None)
gu.addEquivalentClass(g, gene_id, eqid)
elem.clear() # discard the element
if self.testMode and limit is not None and line_counter > limit:
return
gu.loadProperties(
g, G2PAssoc.annotation_properties, G2PAssoc.ANNOTPROP)
gu.loadProperties(g, G2PAssoc.datatype_properties, G2PAssoc.DATAPROP)
gu.loadProperties(g, G2PAssoc.object_properties, G2PAssoc.OBJECTPROP)
gu.loadAllProperties(g)
return
def _process_diseasegene(self, limit):
"""
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
myfile = "/".join((self.rawdir, self.files["disease-gene"]["file"]))
for event, elem in ET.iterparse(myfile):
if elem.tag == "Disorder":
# get the element name and id
# id = elem.get('id') # some internal identifier
disorder_num = elem.find("OrphaNumber").text
disorder_id = "Orphanet:" + str(disorder_num)
if self.testMode and disorder_id not in config.get_config()["test_ids"]["disease"]:
continue
disorder_label = elem.find("Name").text
# make a hash of internal gene id to type for later lookup
gene_iid_to_type = {}
gene_list = elem.find("GeneList")
for gene in gene_list.findall("Gene"):
gene_iid = gene.get("id")
gene_type = gene.find("GeneType").get("id")
gene_iid_to_type[gene_iid] = gene_type
gu.addClassToGraph(g, disorder_id, disorder_label) # assuming that these are in the ontology
assoc_list = elem.find("DisorderGeneAssociationList")
for a in assoc_list.findall("DisorderGeneAssociation"):
gene_iid = a.find(".//Gene").get("id")
gene_name = a.find(".//Gene/Name").text
gene_symbol = a.find(".//Gene/Symbol").text
gene_num = a.find("./Gene/OrphaNumber").text
gene_id = "Orphanet:" + str(gene_num)
gene_type_id = self._map_gene_type_id(gene_iid_to_type[gene_iid])
gu.addClassToGraph(g, gene_id, gene_symbol, gene_type_id, gene_name)
syn_list = a.find("./Gene/SynonymList")
if int(syn_list.get("count")) > 0:
for s in syn_list.findall("./Synonym"):
gu.addSynonym(g, gene_id, s.text)
dgtype = a.find("DisorderGeneAssociationType").get("id")
rel_id = self._map_rel_id(dgtype)
dg_label = a.find("./DisorderGeneAssociationType/Name").text
if rel_id is None:
logger.warn(
"Cannot map association type (%s) to RO for association (%s | %s). Skipping.",
dg_label,
disorder_label,
gene_symbol,
)
continue
alt_locus_id = "_" + gene_num + "-" + disorder_num + "VL"
alt_label = " ".join(
("some variant of", gene_symbol.strip(), "that is a", dg_label.lower(), disorder_label)
)
if self.nobnodes:
alt_locus_id = ":" + alt_locus_id
gu.addIndividualToGraph(g, alt_locus_id, alt_label, geno.genoparts["variant_locus"])
geno.addAlleleOfGene(alt_locus_id, gene_id)
# consider typing the gain/loss-of-function variants like:
# http://sequenceontology.org/browser/current_svn/term/SO:0002054
# http://sequenceontology.org/browser/current_svn/term/SO:0002053
# use "assessed" status to issue an evidence code
# FIXME I think that these codes are sub-optimal
status_code = a.find("DisorderGeneAssociationStatus").get("id")
eco_id = "ECO:0000323" # imported automatically asserted information used in automatic assertion
if status_code == "17991": # Assessed # TODO are these internal ids stable between releases?
eco_id = "ECO:0000322" # imported manually asserted information used in automatic assertion
# Non-traceable author statement ECO_0000034
# imported information in automatic assertion ECO_0000313
assoc = G2PAssoc(self.name, alt_locus_id, disorder_id, rel_id)
assoc.add_evidence(eco_id)
assoc.add_association_to_graph(g)
rlist = a.find("./Gene/ExternalReferenceList")
eqid = None
for r in rlist.findall("ExternalReference"):
if r.find("Source").text == "Ensembl":
eqid = "ENSEMBL:" + r.find("Reference").text
elif r.find("Source").text == "HGNC":
eqid = "HGNC:" + r.find("Reference").text
elif r.find("Source").text == "OMIM":
eqid = "OMIM:" + r.find("Reference").text
else:
pass # skip the others for now
if eqid is not None:
gu.addClassToGraph(g, eqid, None)
gu.addEquivalentClass(g, gene_id, eqid)
pass
elem.clear() # discard the element
if self.testMode and limit is not None and line_counter > limit:
return
gu.loadProperties(g, G2PAssoc.annotation_properties, G2PAssoc.ANNOTPROP)
gu.loadProperties(g, G2PAssoc.datatype_properties, G2PAssoc.DATAPROP)
gu.loadProperties(g, G2PAssoc.object_properties, G2PAssoc.OBJECTPROP)
gu.loadAllProperties(g)
return
def _get_process_allelic_variants(self, entry, graph):
model = Model(graph)
reference = Reference(graph)
geno = Genotype(graph)
if entry is not None:
# to hold the entry-specific publication mentions
# for the allelic variants
publist = {}
entry_num = entry['mimNumber']
# process the ref list just to get the pmids
ref_to_pmid = self._get_pubs(entry, graph)
if 'allelicVariantList' in entry:
for alv in entry['allelicVariantList']:
al_num = alv['allelicVariant']['number']
al_id = 'OMIM:' + str(entry_num) + '.' + str(al_num).zfill(
4)
al_label = None
al_description = None
if alv['allelicVariant']['status'] == 'live':
publist[al_id] = set()
if 'mutations' in alv['allelicVariant']:
al_label = alv['allelicVariant']['mutations']
if 'text' in alv['allelicVariant']:
al_description = alv['allelicVariant']['text']
mch = re.findall(r'\{(\d+)\:', al_description)
publist[al_id] = set(mch)
geno.addAllele(al_id, al_label,
self.globaltt['variant_locus'],
al_description)
geno.addAlleleOfGene(al_id, 'OMIM:' + str(entry_num),
self.globaltt['is_allele_of'])
for ref in publist[al_id]:
pmid = ref_to_pmid[int(ref)]
graph.addTriple(pmid, self.globaltt['is_about'],
al_id)
# look up the pubmed id in the list of references
if 'dbSnps' in alv['allelicVariant']:
dbsnp_ids = re.split(
r',', alv['allelicVariant']['dbSnps'])
for dnum in dbsnp_ids:
did = 'dbSNP:' + dnum.strip()
model.addIndividualToGraph(did, None)
model.addSameIndividual(al_id, did)
# Note that RCVs are variant to disease associations
# in ClinVar, rather than variant entries
# so we make these xrefs instead of equivalents
if 'clinvarAccessions' in alv['allelicVariant']:
# clinvarAccessions triple semicolon delimited
# each >1 like RCV000020059;;;
rcv_ids = \
alv['allelicVariant']['clinvarAccessions'].split(';;;')
rcv_ids = [rcv[:12]
for rcv in rcv_ids] # incase more cruft
for rnum in rcv_ids:
rid = 'ClinVar:' + rnum
model.addXref(al_id, rid)
reference.addPage(
al_id, "http://omim.org/entry/" + '#'.join(
(str(entry_num), str(al_num).zfill(4))))
elif re.search(r'moved', alv['allelicVariant']['status']):
# for both 'moved' and 'removed'
moved_ids = None
if 'movedTo' in alv['allelicVariant']:
moved_id = 'OMIM:' + alv['allelicVariant'][
'movedTo']
moved_ids = [moved_id]
model.addDeprecatedIndividual(al_id, moved_ids)
else:
LOG.error('Uncaught alleleic variant status %s',
alv['allelicVariant']['status'])
def _process_data(self, raw, limit=None):
LOG.info("Processing Data from %s", raw)
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
geno = Genotype(graph)
# Add the taxon as a class
taxon_id = self.globaltt['Mus musculus']
model.addClassToGraph(taxon_id, None)
# with open(raw, 'r', encoding="utf8") as csvfile:
col = self.files['all']['columns']
with gzip.open(raw, 'rt') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='\"')
row = next(reader) # presumed header
if not self.check_fileheader(col, row):
pass
for row in reader:
# | head -1 | tr ',' '\n' | sed "s|\(.*\)|# \1 = row[col.index('\1')]|g"
marker_accession_id = row[col.index('marker_accession_id')].strip()
marker_symbol = row[col.index('marker_symbol')].strip()
phenotyping_center = row[col.index('phenotyping_center')].strip()
colony_raw = row[col.index('colony_id')].strip()
sex = row[col.index('sex')].strip()
zygosity = row[col.index('zygosity')].strip()
allele_accession_id = row[col.index('allele_accession_id')].strip()
allele_symbol = row[col.index('allele_symbol')].strip()
# allele_name = row[col.index('allele_name')]
strain_accession_id = row[col.index('strain_accession_id')].strip()
strain_name = row[col.index('strain_name')].strip()
# project_name = row[col.index('project_name')]
project_fullname = row[col.index('project_fullname')].strip()
pipeline_name = row[col.index('pipeline_name')].strip()
pipeline_stable_id = row[col.index('pipeline_stable_id')].strip()
procedure_stable_id = row[col.index('procedure_stable_id')].strip()
procedure_name = row[col.index('procedure_name')].strip()
parameter_stable_id = row[col.index('parameter_stable_id')].strip()
parameter_name = row[col.index('parameter_name')].strip()
# top_level_mp_term_id = row[col.index('top_level_mp_term_id')]
# top_level_mp_term_name = row[col.index('top_level_mp_term_name')]
mp_term_id = row[col.index('mp_term_id')].strip()
mp_term_name = row[col.index('mp_term_name')].strip()
p_value = row[col.index('p_value')].strip()
percentage_change = row[col.index('percentage_change')].strip()
effect_size = row[col.index('effect_size')].strip()
statistical_method = row[col.index('statistical_method')].strip()
resource_name = row[col.index('resource_name')].strip()
if self.test_mode and marker_accession_id not in self.gene_ids:
continue
# ##### cleanup some of the identifiers ######
zygosity = zygosity.strip()
zygosity_id = self.resolve(zygosity)
if zygosity_id == zygosity:
LOG.warning(
"Zygosity '%s' unmapped. detting to indeterminate", zygosity)
zygosity_id = self.globaltt['indeterminate']
# colony ids sometimes have <> in them, spaces,
# or other non-alphanumerics and break our system;
# replace these with underscores
colony_id = '_:' + re.sub(r'\W+', '_', colony_raw)
if not re.match(r'MGI', allele_accession_id):
allele_accession_id = '_:IMPC-'+re.sub(
r':', '', allele_accession_id)
if re.search(r'EUROCURATE', strain_accession_id):
# the eurocurate links don't resolve at IMPC
# TODO blank nodes do not maintain identifiers
strain_accession_id = '_:' + strain_accession_id
elif not re.match(r'MGI', strain_accession_id):
LOG.info(
"Found a strange strain accession...%s", strain_accession_id)
strain_accession_id = 'IMPC:'+strain_accession_id
######################
# first, add the marker and variant to the graph as with MGI,
# the allele is the variant locus. IF the marker is not known,
# we will call it a sequence alteration. otherwise,
# we will create a BNode for the sequence alteration.
sequence_alteration_id = variant_locus_id = None
variant_locus_name = sequence_alteration_name = None
# extract out what's within the <> to get the symbol
if re.match(r'.*<.*>', allele_symbol):
sequence_alteration_name = re.match(
r'.*<(.*)>', allele_symbol)
if sequence_alteration_name is not None:
sequence_alteration_name = sequence_alteration_name.group(1)
else:
sequence_alteration_name = allele_symbol
if marker_accession_id is not None and marker_accession_id == '':
LOG.warning("Marker unspecified on row %d", reader.line_num)
marker_accession_id = None
if marker_accession_id is not None:
variant_locus_id = allele_accession_id
variant_locus_name = allele_symbol
variant_locus_type = self.globaltt['variant_locus']
geno.addGene(
marker_accession_id, marker_symbol, self.globaltt['gene'])
geno.addAllele(
variant_locus_id, variant_locus_name, variant_locus_type, None)
geno.addAlleleOfGene(variant_locus_id, marker_accession_id)
# TAG bnode
sequence_alteration_id = '_:seqalt' + re.sub(
r':', '', allele_accession_id)
geno.addSequenceAlterationToVariantLocus(
sequence_alteration_id, variant_locus_id)
else:
sequence_alteration_id = allele_accession_id
# IMPC contains targeted mutations with either gene traps,
# knockouts, insertion/intragenic deletions.
# but I don't really know what the SeqAlt is here,
# so I don't add it.
geno.addSequenceAlteration(
sequence_alteration_id, sequence_alteration_name)
# ############# BUILD THE COLONY #############
# First, let's describe the colony that the animals come from
# The Colony ID refers to the ES cell clone
# used to generate a mouse strain.
# Terry sez: we use this clone ID to track
# ES cell -> mouse strain -> mouse phenotyping.
# The same ES clone maybe used at multiple centers,
# so we have to concatenate the two to have a unique ID.
# some useful reading about generating mice from ES cells:
# http://ki.mit.edu/sbc/escell/services/details
# here, we'll make a genotype
# that derives from an ES cell with a given allele.
# the strain is not really attached to the colony.
# the colony/clone is reflective of the allele, with unknown zygosity
stem_cell_class = self.globaltt['embryonic stem cell line']
if colony_id is None:
print(colony_raw, stem_cell_class, "\nline:\t", reader.line_num)
model.addIndividualToGraph(colony_id, colony_raw, stem_cell_class)
# vslc of the colony has unknown zygosity
# note that we will define the allele
# (and it's relationship to the marker, etc.) later
# FIXME is it really necessary to create this vslc
# when we always know it's unknown zygosity?
vslc_colony = '_:'+re.sub(
r':', '', allele_accession_id + self.globaltt['indeterminate'])
vslc_colony_label = allele_symbol + '/<?>'
# for ease of reading, we make the colony genotype variables.
# in the future, it might be desired to keep the vslcs
colony_genotype_id = vslc_colony
colony_genotype_label = vslc_colony_label
geno.addGenotype(colony_genotype_id, colony_genotype_label)
geno.addParts(
allele_accession_id, colony_genotype_id,
self.globaltt['has_variant_part'])
geno.addPartsToVSLC(
vslc_colony, allele_accession_id, None,
self.globaltt['indeterminate'], self.globaltt['has_variant_part'])
graph.addTriple(
colony_id, self.globaltt['has_genotype'], colony_genotype_id)
# ########## BUILD THE ANNOTATED GENOTYPE ##########
# now, we'll build the genotype of the individual that derives
# from the colony/clone genotype that is attached to
# phenotype = colony_id + strain + zygosity + sex
# (and is derived from a colony)
# this is a sex-agnostic genotype
genotype_id = self.make_id(
(colony_id + phenotyping_center + zygosity + strain_accession_id))
geno.addSequenceDerivesFrom(genotype_id, colony_id)
# build the VSLC of the sex-agnostic genotype
# based on the zygosity
allele1_id = allele_accession_id
allele2_id = allele2_rel = None
allele1_label = allele_symbol
allele2_label = '<?>'
# Making VSLC labels from the various parts,
# can change later if desired.
if zygosity == 'heterozygote':
allele2_label = re.sub(r'<.*', '<+>', allele1_label)
allele2_id = None
elif zygosity == 'homozygote':
allele2_label = allele1_label
allele2_id = allele1_id
allele2_rel = self.globaltt['has_variant_part']
elif zygosity == 'hemizygote':
allele2_label = re.sub(r'<.*', '<0>', allele1_label)
allele2_id = None
elif zygosity == 'not_applicable':
allele2_label = re.sub(r'<.*', '<?>', allele1_label)
allele2_id = None
else:
LOG.warning("found unknown zygosity %s", zygosity)
break
vslc_name = '/'.join((allele1_label, allele2_label))
# Add the VSLC
vslc_id = '-'.join(
(marker_accession_id, allele_accession_id, zygosity))
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:'+vslc_id
model.addIndividualToGraph(
vslc_id, vslc_name,
self.globaltt['variant single locus complement'])
geno.addPartsToVSLC(
vslc_id, allele1_id, allele2_id, zygosity_id,
self.globaltt['has_variant_part'], allele2_rel)
# add vslc to genotype
geno.addVSLCtoParent(vslc_id, genotype_id)
# note that the vslc is also the gvc
model.addType(vslc_id, self.globaltt['genomic_variation_complement'])
# Add the genomic background
# create the genomic background id and name
if strain_accession_id != '':
genomic_background_id = strain_accession_id
else:
genomic_background_id = None
genotype_name = vslc_name
if genomic_background_id is not None:
geno.addGenotype(
genomic_background_id, strain_name,
self.globaltt['genomic_background'])
# make a phenotyping-center-specific strain
# to use as the background
pheno_center_strain_label = strain_name + '-' + phenotyping_center \
+ '-' + colony_raw
pheno_center_strain_id = '-'.join((
re.sub(r':', '', genomic_background_id),
re.sub(r'\s', '_', phenotyping_center),
re.sub(r'\W+', '', colony_raw)))
if not re.match(r'^_', pheno_center_strain_id):
# Tag bnode
pheno_center_strain_id = '_:' + pheno_center_strain_id
geno.addGenotype(
pheno_center_strain_id, pheno_center_strain_label,
self.globaltt['genomic_background'])
geno.addSequenceDerivesFrom(
pheno_center_strain_id, genomic_background_id)
# Making genotype labels from the various parts,
# can change later if desired.
# since the genotype is reflective of the place
# it got made, should put that in to disambiguate
genotype_name = \
genotype_name + ' [' + pheno_center_strain_label + ']'
geno.addGenomicBackgroundToGenotype(
pheno_center_strain_id, genotype_id)
geno.addTaxon(taxon_id, pheno_center_strain_id)
# this is redundant, but i'll keep in in for now
geno.addSequenceDerivesFrom(genotype_id, colony_id)
geno.addGenotype(genotype_id, genotype_name)
# Make the sex-qualified genotype,
# which is what the phenotype is associated with
sex_qualified_genotype_id = \
self.make_id((
colony_id + phenotyping_center + zygosity +
strain_accession_id + sex))
sex_qualified_genotype_label = genotype_name + ' (' + sex + ')'
sq_type_id = self.resolve(sex, False)
if sq_type_id == sex:
sq_type_id = self.globaltt['intrinsic_genotype']
LOG.warning(
"Unknown sex qualifier %s, adding as intrinsic_genotype",
sex)
geno.addGenotype(
sex_qualified_genotype_id, sex_qualified_genotype_label, sq_type_id)
geno.addParts(
genotype_id, sex_qualified_genotype_id,
self.globaltt['has_variant_part'])
if genomic_background_id is not None and genomic_background_id != '':
# Add the taxon to the genomic_background_id
geno.addTaxon(taxon_id, genomic_background_id)
else:
# add it as the genomic background
geno.addTaxon(taxon_id, genotype_id)
# ############# BUILD THE G2P ASSOC #############
# from an old email dated July 23 2014:
# Phenotypes associations are made to
# imits colony_id+center+zygosity+gender
# sometimes phenotype ids are missing. (about 711 early 2020)
if mp_term_id is None or mp_term_id == '':
LOG.warning(
"No phenotype id specified for row %d", reader.line_num)
continue
# hard coded ECO code
eco_id = self.globaltt['mutant phenotype evidence']
# the association comes as a result of a g2p from
# a procedure in a pipeline at a center and parameter tested
assoc = G2PAssoc(
graph, self.name, sex_qualified_genotype_id, mp_term_id)
assoc.add_evidence(eco_id)
# assoc.set_score(float(p_value))
# TODO add evidence instance using
# pipeline_stable_id +
# procedure_stable_id +
# parameter_stable_id
assoc.add_association_to_graph()
assoc_id = assoc.get_association_id()
model._addSexSpecificity(assoc_id, self.resolve(sex))
# add a free-text description
try:
description = ' '.join((
mp_term_name, 'phenotype determined by', phenotyping_center,
'in an', procedure_name, 'assay where', parameter_name.strip(),
'was measured with an effect_size of',
str(round(float(effect_size), 5)),
'(p =', "{:.4e}".format(float(p_value)), ').'))
except ValueError:
description = ' '.join((
mp_term_name, 'phenotype determined by', phenotyping_center,
'in an', procedure_name, 'assay where', parameter_name.strip(),
'was measured with an effect_size of', str(effect_size),
'(p =', "{0}".format(p_value), ').'))
study_bnode = self._add_study_provenance(
phenotyping_center, colony_raw, project_fullname, pipeline_name,
pipeline_stable_id, procedure_stable_id, procedure_name,
parameter_stable_id, parameter_name, statistical_method,
resource_name)
evidence_line_bnode = self._add_evidence(
assoc_id, eco_id, p_value, percentage_change, effect_size,
study_bnode)
self._add_assertion_provenance(assoc_id, evidence_line_bnode)
model.addDescription(evidence_line_bnode, description)
# resource_id = resource_name
# assoc.addSource(graph, assoc_id, resource_id)
if not self.test_mode and limit is not None and reader.line_num > limit:
break
def _process_QTLs_genetic_location(self, raw, taxon_id, common_name, limit=None):
"""
This function processes
Triples created:
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
eco_id = "ECO:0000061" # Quantitative Trait Analysis Evidence
logger.info("Processing genetic location for %s", taxon_id)
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(qtl_id, qtl_symbol, trait_name, assotype, empty, chromosome, position_cm, range_cm,
flankmark_a2, flankmark_a1, peak_mark, flankmark_b1, flankmark_b2, exp_id, model, test_base,
sig_level, lod_score, ls_mean, p_values, f_statistics, variance, bayes_value, likelihood_ratio,
trait_id, dom_effect, add_effect, pubmed_id, gene_id, gene_id_src, gene_id_type, empty2) = row
if self.testMode and int(qtl_id) not in self.test_ids:
continue
qtl_id = 'AQTL:'+qtl_id
trait_id = 'AQTLTrait:'+trait_id
# Add QTL to graph
f = Feature(qtl_id, qtl_symbol, geno.genoparts['QTL'])
f.addTaxonToFeature(g, taxon_id)
# deal with the chromosome
chrom_id = makeChromID(chromosome, taxon_id, 'CHR')
# add a version of the chromosome which is defined as the genetic map
build_id = 'MONARCH:'+common_name.strip()+'-linkage'
build_label = common_name+' genetic map'
geno.addReferenceGenome(build_id, build_label, taxon_id)
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(chromosome, build_id, build_label, chrom_id)
start = stop = None
if re.search('-', range_cm):
range_parts = re.split('-', range_cm)
# check for poorly formed ranges
if len(range_parts) == 2 and range_parts[0] != '' and range_parts[1] != '':
(start, stop) = [int(float(x.strip())) for x in re.split('-', range_cm)]
else:
logger.info("There's a cM range we can't handle for QTL %s: %s", qtl_id, range_cm)
elif position_cm != '':
start = stop = int(float(position_cm))
# FIXME remove converion to int for start/stop when schema can handle floats
# add in the genetic location based on the range
f.addFeatureStartLocation(start, chrom_in_build_id, None, [Feature.types['FuzzyPosition']])
f.addFeatureEndLocation(stop, chrom_in_build_id, None, [Feature.types['FuzzyPosition']])
f.addFeatureToGraph(g)
# sometimes there's a peak marker, like a rsid. we want to add that as a variant of the gene,
# and xref it to the qtl.
dbsnp_id = None
if peak_mark != '' and peak_mark != '.' and re.match('rs', peak_mark.strip()):
dbsnp_id = 'dbSNP:'+peak_mark.strip()
gu.addIndividualToGraph(g, dbsnp_id, None, geno.genoparts['sequence_alteration'])
gu.addXref(g, qtl_id, dbsnp_id)
if gene_id is not None and gene_id != '' and gene_id != '.':
if gene_id_src == 'NCBIgene' or gene_id_src == '': # we assume if no src is provided, it's NCBI
gene_id = 'NCBIGene:'+gene_id.strip()
geno.addGene(gene_id, None) # we will expect that these labels provided elsewhere
geno.addAlleleOfGene(qtl_id, gene_id, geno.object_properties['feature_to_gene_relation']) # FIXME what is the right relationship here?
if dbsnp_id is not None:
# add the rsid as a seq alt of the gene_id
vl_id = '_' + re.sub(':', '', gene_id) + '-' + peak_mark
if self.nobnodes:
vl_id = ':' + vl_id
geno.addSequenceAlterationToVariantLocus(dbsnp_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
# add the trait
gu.addClassToGraph(g, trait_id, trait_name)
# Add publication
r = None
if re.match('ISU.*', pubmed_id):
pub_id = 'AQTLPub:'+pubmed_id.strip()
r = Reference(pub_id)
elif pubmed_id != '':
pub_id = 'PMID:'+pubmed_id.strip()
r = Reference(pub_id, Reference.ref_types['journal_article'])
if r is not None:
r.addRefToGraph(g)
# make the association to the QTL
assoc = G2PAssoc(self.name, qtl_id, trait_id, gu.object_properties['is_marker_for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id as evidence
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
score = float(re.sub('<', '', p_values))
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph(g)
# make the association to the dbsnp_id, if found
if dbsnp_id is not None:
# make the association to the dbsnp_id
assoc = G2PAssoc(self.name, dbsnp_id, trait_id, gu.object_properties['is_marker_for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
score = float(re.sub('<', '', p_values))
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph(g)
if not self.testMode and limit is not None and line_counter > limit:
break
logger.info("Done with QTL genetic info")
return
def process_allele_phenotype(self, limit=None):
"""
This file compactly lists variant to phenotype associations,
such that in a single row, there may be >1 variant listed
per phenotype and paper. This indicates that each variant is
individually assocated with the given phenotype,
as listed in 1+ papers.
(Not that the combination of variants is producing the phenotype.)
:param limit:
:return:
"""
raw = '/'.join((self.rawdir, self.files['allele_pheno']['file']))
if self.testMode:
g = self.testgraph
else:
g = self.graph
# gu = GraphUtils(curie_map.get()) # TODO unused
logger.info("Processing Allele phenotype associations")
line_counter = 0
geno = Genotype(g)
with open(raw, 'r') as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
if re.match(r'!', ''.join(row)): # header
continue
line_counter += 1
(db, gene_num, gene_symbol, is_not, phenotype_id, ref,
eco_symbol, with_or_from, aspect, gene_name, gene_synonym,
gene_class, taxon, date, assigned_by, blank, blank2) = row
if self.testMode and gene_num not in self.test_ids['gene']:
continue
# TODO add NOT phenotypes
if is_not == 'NOT':
continue
eco_id = None
if eco_symbol == 'IMP':
eco_id = 'ECO:0000015'
elif eco_symbol.strip() != '':
logger.warning(
"Encountered an ECO code we don't have: %s",
eco_symbol)
# according to the GOA spec, persons are not allowed to be
# in the reference column, therefore they the variant and
# persons are swapped between the reference and with column.
# we unswitch them here.
temp_var = temp_ref = None
if re.search(r'WBVar|WBRNAi', ref):
temp_var = ref
# move the paper from the with column into the ref
if re.search(r'WBPerson', with_or_from):
temp_ref = with_or_from
if temp_var is not None or temp_ref is not None:
with_or_from = temp_var
ref = temp_ref
allele_list = re.split(r'\|', with_or_from)
if len(allele_list) == 0:
logger.error(
"Missing alleles from phenotype assoc at line %d",
line_counter)
continue
else:
for a in allele_list:
allele_num = re.sub(r'WB:', '', a.strip())
allele_id = 'WormBase:'+allele_num
gene_id = 'WormBase:'+gene_num
if re.search(r'WBRNAi', allele_id):
# make the reagent-targeted gene,
# & annotate that instead of the RNAi item directly
rnai_num = re.sub(r'WormBase:', '', allele_id)
rnai_id = allele_id
rtg_id = self.make_reagent_targeted_gene_id(
gene_num, rnai_num, self.nobnodes)
geno.addReagentTargetedGene(
rnai_id, 'WormBase:'+gene_num, rtg_id)
geno.addGeneTargetingReagent(
rnai_id, None, geno.genoparts['RNAi_reagent'],
gene_id)
allele_id = rtg_id
elif re.search(r'WBVar', allele_id):
# this may become deprecated by using wormmine
# make the allele to gene relationship
# the WBVars are really sequence alterations
# the public name will come from elsewhere
geno.addSequenceAlteration(allele_id, None)
vl_id = '_'+'-'.join((gene_num, allele_num))
if self.nobnodes:
vl_id = ':'+vl_id
geno.addSequenceAlterationToVariantLocus(
allele_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
logger.warning(
"Some kind of allele I don't recognize: %s",
allele_num)
continue
assoc = G2PAssoc(self.name, allele_id, phenotype_id)
if eco_id is not None:
assoc.add_evidence(eco_id)
if ref is not None and ref != '':
ref = re.sub(r'(WB:|WB_REF:)', 'WormBase:', ref)
r = Reference(ref)
if re.search(r'Person', ref):
r.setType(r.ref_types['person'])
# also add
# inferred from background scientific knowledge
assoc.add_evidence('ECO:0000001')
r.addRefToGraph(g)
assoc.add_source(ref)
assoc.add_association_to_graph(g)
# finish looping through all alleles
if not self.testMode \
and limit is not None and line_counter > limit:
break
return
def _process_kegg_disease2gene(self, limit=None):
"""
This method creates an association between diseases and their associated genes.
We are being conservative here, and only processing those diseases for which there
is no mapping to OMIM.
Triples created:
<alternate_locus> is an Individual
<alternate_locus> has type <variant_locus>
<alternate_locus> is an allele of <gene_id>
<assoc_id> has subject <disease_id>
<assoc_id> has object <gene_id>
:param limit:
:return:
"""
logger.info("Processing KEGG disease to gene")
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
rel = gu.object_properties['is_marker_for']
gu.loadAllProperties(g)
noomimset = set()
raw = '/'.join((self.rawdir, self.files['disease_gene']['file']))
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(gene_id, disease_id) = row
if self.testMode and gene_id not in self.test_ids['genes']:
continue
gene_id = 'KEGG-'+gene_id.strip()
disease_id = 'KEGG-'+disease_id.strip()
# only add diseases for which there is no omim id and not a grouping class
if disease_id not in self.kegg_disease_hash:
# add as a class
disease_label = None
if disease_id in self.label_hash:
disease_label = self.label_hash[disease_id]
if re.search('includ', str(disease_label)):
# they use 'including' when it's a grouping class
logger.info("Skipping this association because it's a grouping class: %s", disease_label)
continue
gu.addClassToGraph(g, disease_id, disease_label, 'DOID:4') # type this disease_id as a disease
noomimset.add(disease_id)
alt_locus_id = self._make_variant_locus_id(gene_id, disease_id)
alt_label = self.label_hash[alt_locus_id]
gu.addIndividualToGraph(g, alt_locus_id, alt_label, geno.genoparts['variant_locus'])
geno.addAlleleOfGene(alt_locus_id, gene_id)
# Add the disease to gene relationship.
assoc = G2PAssoc(self.name, alt_locus_id, disease_id, rel)
assoc.load_all_properties(g)
assoc.add_association_to_graph(g)
if (not self.testMode) and (limit is not None and line_counter > limit):
break
logger.info("Done with KEGG disease to gene")
logger.info("Found %d diseases with no omim id", len(noomimset))
return
def _process_omim2gene(self, limit=None):
"""
This method maps the OMIM IDs and KEGG gene ID. Currently split based on the link_type field.
Equivalent link types are mapped as gene XRefs.
Reverse link types are mapped as disease to gene associations.
Original link types are currently skipped.
Triples created:
<kegg_gene_id> is a Gene
<omim_gene_id> is a Gene
<kegg_gene_id>> hasXref <omim_gene_id>
<assoc_id> has subject <omim_disease_id>
<assoc_id> has object <kegg_gene_id>
:param limit:
:return:
"""
logger.info("Processing OMIM to KEGG gene")
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
raw = '/'.join((self.rawdir, self.files['omim2gene']['file']))
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(kegg_gene_id, omim_id, link_type) = row
if self.testMode and kegg_gene_id not in self.test_ids['genes']:
continue
kegg_gene_id = 'KEGG-'+kegg_gene_id.strip()
omim_id = re.sub('omim', 'OMIM', omim_id)
if link_type == 'equivalent':
# these are genes! so add them as a class then make equivalence
gu.addClassToGraph(g, omim_id, None)
geno.addGene(kegg_gene_id, None)
gu.addEquivalentClass(g, kegg_gene_id, omim_id)
elif link_type == 'reverse':
# make an association between an OMIM ID and the KEGG gene ID
# we do this with omim ids because they are more atomic than KEGG ids
alt_locus_id = self._make_variant_locus_id(kegg_gene_id, omim_id)
alt_label = self.label_hash[alt_locus_id]
gu.addIndividualToGraph(g, alt_locus_id, alt_label, geno.genoparts['variant_locus'])
geno.addAlleleOfGene(alt_locus_id, kegg_gene_id)
# Add the disease to gene relationship.
rel = gu.object_properties['is_marker_for']
assoc = G2PAssoc(self.name, alt_locus_id, omim_id, rel)
assoc.add_association_to_graph(g)
elif link_type == 'original':
# these are sometimes a gene, and sometimes a disease
logger.info('Unable to handle original link for %s-%s', kegg_gene_id, omim_id)
else:
# don't know what these are
logger.warn('Unhandled link type for %s-%s: %s', kegg_gene_id, omim_id, link_type)
if (not self.testMode) and (limit is not None and line_counter > limit):
break
logger.info("Done with OMIM to KEGG gene")
gu.loadProperties(g, G2PAssoc.annotation_properties, G2PAssoc.ANNOTPROP)
gu.loadProperties(g, G2PAssoc.datatype_properties, G2PAssoc.DATAPROP)
gu.loadProperties(g, G2PAssoc.object_properties, G2PAssoc.OBJECTPROP)
return
def _process_morbidmap(self, limit):
"""
This will process the morbidmap file to get the links between omim genes and diseases.
Here, we create anonymous nodes for some variant loci that are variants of the gene that causes the disease.
Triples created:
<some_anonymous_variant_locus> is_sequence_variant_instance_of <omim_gene_id>
<some_anonymous_variant_locus> has_phenotype <omim_disease_id>
<assoc> hasSubject <some_anonymous_variant_locus>
<assoc> hasObject <omim_disease_id>
<assoc> hasPredicate <has_phenotype>
<assoc> DC:evidence <eco_id>
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
with open('/'.join((self.rawdir, self.files['morbidmap']['file']))) as f:
for line in f:
line = line.strip()
line_counter += 1
(disorder, gene_symbols, gene_num, loc) = line.split('|')
# disorder = disorder label , number (mapping key)
# 3-M syndrome 1, 273750 (3)|CUL7, 3M1|609577|6p21.1
# but note that for those diseases where they are genomic loci (not genes though),
# the omim id is only listed as the gene
# Alopecia areata 1 (2)|AA1|104000|18p11.3-p11.2
disorder_match = re.match('(.*), (\d+) \((\d+)\)', disorder)
if disorder_match is not None:
disorder_parts = disorder_match.groups()
if len(disorder_parts) == 3:
(disorder_label, disorder_num, phene_key) = disorder_parts
else:
logger.warn("I couldn't parse disorder string: %s", disorder)
continue
if self.testMode and (int(disorder_num) not in self.test_ids or int(gene_num) not in self.test_ids):
continue
gene_symbols = gene_symbols.split(', ')
gene_id = ':'.join(('OMIM', gene_num))
disorder_id = ':'.join(('OMIM', disorder_num))
rel_id = gu.object_properties['has_phenotype'] # default
rel_label = 'causes'
if re.match('\[', disorder_label):
rel_id = gu.object_properties['is_marker_for']
rel_label = 'is a marker for'
elif re.match('\{', disorder_label):
rel_id = gu.object_properties['contributes_to']
rel_label = 'contributes to'
evidence = self._map_phene_mapping_code_to_eco(phene_key)
# we actually want the association between the gene and the disease to be via an alternate locus
# not the "wildtype" gene itself.
# so we make an anonymous alternate locus, and put that in the association.
alt_locus = '_'+gene_num+'-'+disorder_num+'VL'
alt_label = gene_symbols[0].strip()
if alt_label is not None and alt_label != '':
alt_label = ' '.join(('some variant of', alt_label.strip(),
'that', rel_label, disorder_label))
else:
alt_label = None
gu.addIndividualToGraph(g, alt_locus, alt_label, geno.genoparts['variant_locus'])
geno.addAlleleOfGene(alt_locus, gene_id)
assoc = G2PAssoc(self.name, alt_locus, disorder_id, rel_id)
assoc.add_evidence(evidence)
assoc.load_all_properties(g)
assoc.add_association_to_graph(g)
if not self.testMode and limit is not None and line_counter > limit:
break
gu.loadProperties(g, geno.object_properties, gu.OBJPROP)
return
