def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
gu = GraphUtils(curie_map.get())
fname = '/'.join((self.rawdir, self.files['catalog']['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = 'CL:0000034'
mouse_taxon = 'NCBITaxon:10090'
geno = Genotype(g)
with open(fname, 'r', encoding="utf8") as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
for row in filereader:
line_counter += 1
# skip the first 3 lines which are header, etc.
if line_counter < 4:
continue
(strain_id, strain_label, strain_type_symbol, strain_state,
mgi_allele_id, mgi_allele_symbol, mgi_allele_name,
mutation_type, chrom, mgi_gene_id, mgi_gene_symbol,
mgi_gene_name, sds_url, accepted_date, mp_ids, pubmed_nums,
research_areas) = row
if self.testMode and (strain_id not in self.test_ids):
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {'variants': set(),
'genes': set()}
# clean up the bad one
if mgi_allele_id == 'multiple mutation':
logger.error("Erroneous gene id: %s", mgi_allele_id)
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the
# sequence alteration types
# var_type =
# self._get_variant_type_from_abbrev(mutation_type)
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id,
# mgi_allele_id)
# scrub out any spaces
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id.strip() != '':
if re.match(r'Gene\s*ID:', mgi_gene_id, re.I):
mgi_gene_id = re.sub(r'Gene\s*ID:\s*', 'NCBIGene:',
mgi_gene_id)
elif not re.match(r'MGI', mgi_gene_id):
logger.info("Gene id not recognized: %s", mgi_gene_id)
if re.match(r'\d+$', mgi_gene_id):
# assume that if it's all numbers, then it's MGI
mgi_gene_id = 'MGI:'+str(mgi_gene_id)
logger.info("Assuming numerics are MGI.")
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors -
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol.strip() != '' and mgi_gene_id == '':
logger.error(
"Gene label with no identifier for strain %s: %s",
strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol.strip())
# make a temp id for genes that aren't identified
# tmp_gene_id = '_'+mgi_gene_symbol
# self.id_label_hash[tmp_gene_id] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mp_ids are now a comma delimited list
# with MP terms in brackets
phenotype_ids = []
if mp_ids != '':
for i in re.split(r',', mp_ids):
i = i.strip()
mps = re.search(r'\[(.*)\]', i)
if mps is not None:
mp_id = mps.group(1).strip()
phenotype_ids.append(mp_id)
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums.strip() != '':
for i in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:'+i.strip()
pubmed_ids.append(pmid)
r = Reference(pmid,
Reference.ref_types['journal_article'])
r.addRefToGraph(g)
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
gu.addClassToGraph(g, mouse_taxon, None)
if research_areas.strip() == '':
research_areas = None
else:
research_areas = 'Research Areas: '+research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
gu.addIndividualToGraph(
g, strain_id, strain_label, strain_type,
research_areas) # an inst of mouse??
gu.makeLeader(g, strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in the ontology
gu.addClassToGraph(g, pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(self.name, mgi_allele_id, pid,
gu.object_properties['has_phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph(g)
else:
logger.info("Phenotypes and no allele for %s",
strain_id)
if not self.testMode and (
limit is not None and line_counter > limit):
break
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if len(variants) > 0:
for v in variants:
vl_id = v
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_'+gene+'-VL'
vl_id = re.sub(r':', '', vl_id)
if self.nobnodes:
vl_id = ':'+vl_id
vl_symbol = self.id_label_hash[gene]+'<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = '_'+re.sub(r'^_', '', vl)+'U'
vslc_id = re.sub(r':', '', vslc_id)
if self.nobnodes:
vslc_id = ':' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(
vslc_id, vl, None, geno.zygosity['indeterminate'],
geno.object_properties['has_alternate_part'], None)
gu.addIndividualToGraph(
g, vslc_id, vslc_label,
geno.genoparts['variant_single_locus_complement'])
if len(vslc_list) > 0:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r':', '', gvc_id)
if self.nobnodes:
gvc_id = ':'+gvc_id
gvc_label = \
'; '.join(self.id_label_hash[v] for v in vslc_list)
gu.addIndividualToGraph(
g, gvc_id, gvc_label,
geno.genoparts['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = \
'_' + re.sub(r':', '', '-'.join(
(geno.genoparts['unspecified_genomic_background'],
s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
if self.nobnodes:
bkgd_id = ':'+bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified ('+s+')',
geno.genoparts['unspecified_genomic_background'],
"A placeholder for the " +
"unspecified genetic background for "+s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
geno.genoparts['unspecified_genomic_background'])
geno.addParts(
gvc_id, genotype_id,
geno.object_properties['has_alternate_part'])
geno.addGenotype(genotype_id, genotype_label)
gu.addTriple(
g, s, geno.object_properties['has_genotype'],
genotype_id)
else:
# logger.debug(
# "Strain %s is not making a proper genotype.", s)
pass
gu.loadProperties(
g, G2PAssoc.object_properties, G2PAssoc.OBJECTPROP)
gu.loadProperties(
g, G2PAssoc.datatype_properties, G2PAssoc.DATAPROP)
gu.loadProperties(
g, G2PAssoc.annotation_properties, G2PAssoc.ANNOTPROP)
gu.loadAllProperties(g)
logger.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
return
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
line_counter = 0
fname = '/'.join((self.rawdir, self.files['catalog']['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = 'CL:0000034'
mouse_taxon = 'NCBITaxon:10090'
geno = Genotype(g)
with open(fname, 'r', encoding="utf8") as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
for row in filereader:
line_counter += 1
# skip the first 3 lines which are header, etc.
if line_counter < 4:
continue
(strain_id, strain_label, strain_type_symbol, strain_state,
mgi_allele_id, mgi_allele_symbol, mgi_allele_name,
mutation_type, chrom, mgi_gene_id, mgi_gene_symbol,
mgi_gene_name, sds_url, accepted_date, mp_ids, pubmed_nums,
research_areas) = row
if self.testMode and (strain_id not in self.test_ids) \
or mgi_gene_name == 'withdrawn':
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {
'variants': set(),
'genes': set()
}
# clean up the bad one
if mgi_allele_id == 'multiple mutation':
logger.error("Erroneous gene id: %s", mgi_allele_id)
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the
# sequence alteration types
# var_type =
# self._get_variant_type_from_abbrev(mutation_type)
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id,
# mgi_allele_id)
# scrub out any spaces
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id.strip() != '':
if re.match(r'Gene\s*ID:', mgi_gene_id, re.I):
mgi_gene_id = re.sub(r'Gene\s*ID:\s*', 'NCBIGene:',
mgi_gene_id)
elif not re.match(r'MGI', mgi_gene_id):
logger.info("Gene id not recognized: %s", mgi_gene_id)
if re.match(r'\d+$', mgi_gene_id):
# assume that if it's all numbers, then it's MGI
mgi_gene_id = 'MGI:' + str(mgi_gene_id)
logger.info("Assuming numerics are MGI.")
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors -
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol.strip() != '' and mgi_gene_id == '':
logger.error(
"Gene label with no identifier for strain %s: %s",
strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol.strip())
# make a temp id for genes that aren't identified
# tmp_gene_id = '_'+mgi_gene_symbol
# self.id_label_hash[tmp_gene_id] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mp_ids are now a comma delimited list
# with MP terms in brackets
phenotype_ids = []
if mp_ids != '':
for i in re.split(r',', mp_ids):
i = i.strip()
mps = re.search(r'\[(.*)\]', i)
if mps is not None:
mp_id = mps.group(1).strip()
phenotype_ids.append(mp_id)
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums.strip() != '':
for i in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:' + i.strip()
pubmed_ids.append(pmid)
r = Reference(g, pmid,
Reference.ref_types['journal_article'])
r.addRefToGraph()
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
model.addClassToGraph(mouse_taxon, None)
if research_areas.strip() == '':
research_areas = None
else:
research_areas = 'Research Areas: ' + research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
model.addIndividualToGraph(
strain_id, strain_label, strain_type,
research_areas) # an inst of mouse??
model.makeLeader(strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in the ontology
model.addClassToGraph(pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(
g, self.name, mgi_allele_id, pid,
model.object_properties['has_phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph()
else:
logger.info("Phenotypes and no allele for %s",
strain_id)
if not self.testMode and (limit is not None
and line_counter > limit):
break
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if len(variants) > 0:
for v in variants:
vl_id = v
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_:' + re.sub(r':', '', gene) + '-VL'
vl_symbol = self.id_label_hash[gene] + '<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(vl_id, vl_symbol,
geno.genoparts['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = re.sub(r'^_', '', vl) + 'U'
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(
vslc_id, vl, None, geno.zygosity['indeterminate'],
geno.object_properties['has_alternate_part'], None)
model.addIndividualToGraph(
vslc_id, vslc_label,
geno.genoparts['variant_single_locus_complement'])
if len(vslc_list) > 0:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r'_|:', '', gvc_id)
gvc_id = '_:' + gvc_id
gvc_label = \
'; '.join(self.id_label_hash[v] for v in vslc_list)
model.addIndividualToGraph(
gvc_id, gvc_label,
geno.genoparts['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = \
re.sub(r':', '', '-'.join(
(geno.genoparts['unspecified_genomic_background'],
s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
bkgd_id = '_:' + bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified (' + s + ')',
geno.genoparts['unspecified_genomic_background'],
"A placeholder for the " +
"unspecified genetic background for " + s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
geno.genoparts['unspecified_genomic_background'])
geno.addParts(gvc_id, genotype_id,
geno.object_properties['has_alternate_part'])
geno.addGenotype(genotype_id, genotype_label)
g.addTriple(s, geno.object_properties['has_genotype'],
genotype_id)
else:
# logger.debug(
# "Strain %s is not making a proper genotype.", s)
pass
logger.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
return
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
src_key = 'catalog'
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
fname = '/'.join((self.rawdir, self.files[src_key]['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = self.globaltt['stem cell']
mouse_taxon = self.globaltt['Mus musculus']
geno = Genotype(graph)
with open(fname, 'r', encoding="utf8") as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='\"')
# First line is header not date/version info. This changed recently,
# apparently as of Sep 2019. Also, 3rd line is no longer blank.
row = [x.strip() for x in next(reader)] # messy messy
col = self.files['catalog']['columns']
strain_missing_allele = [] # to count the ones w/insufficent info
if not self.check_fileheader(col, row):
pass
for row in reader:
strain_id = row[col.index('STRAIN/STOCK_ID')].strip()
strain_label = row[col.index('STRAIN/STOCK_DESIGNATION')]
# strain_type_symbol = row[col.index('STRAIN_TYPE')]
strain_state = row[col.index('STATE')]
mgi_allele_id = row[col.index(
'MGI_ALLELE_ACCESSION_ID')].strip()
mgi_allele_symbol = row[col.index('ALLELE_SYMBOL')]
# mgi_allele_name = row[col.index('ALLELE_NAME')]
# mutation_type = row[col.index('MUTATION_TYPE')]
# chrom = row[col.index('CHROMOSOME')]
mgi_gene_id = row[col.index('MGI_GENE_ACCESSION_ID')].strip()
mgi_gene_symbol = row[col.index('GENE_SYMBOL')].strip()
mgi_gene_name = row[col.index('GENE_NAME')]
# sds_url = row[col.index('SDS_URL')]
# accepted_date = row[col.index('ACCEPTED_DATE')]
mpt_ids = row[col.index('MPT_IDS')].strip()
pubmed_nums = row[col.index('PUBMED_IDS')].strip()
research_areas = row[col.index('RESEARCH_AREAS')].strip()
if self.test_mode and (strain_id not in self.test_ids) \
or mgi_gene_name == 'withdrawn':
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {
'variants': set(),
'genes': set()
}
# flag bad ones
if mgi_allele_id[:4] != 'MGI:' and mgi_allele_id != '':
LOG.error("Erroneous MGI allele id: %s", mgi_allele_id)
if mgi_allele_id[:3] == 'MG:':
mgi_allele_id = 'MGI:' + mgi_allele_id[3:]
else:
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the sequence alteration types
# var_type = self.localtt[mutation_type]
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id, mgi_allele_id)
# scrub out any spaces, fix known issues
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id == 'NULL':
mgi_gene_id = ''
elif mgi_gene_id[:7] == 'GeneID:':
mgi_gene_id = 'NCBIGene:' + mgi_gene_id[7:]
if mgi_gene_id != '':
try:
[curie, localid] = mgi_gene_id.split(':')
except ValueError as verror:
LOG.warning(
"Problem parsing mgi_gene_id %s from file %s: %s",
mgi_gene_id, fname, verror)
if curie not in ['MGI', 'NCBIGene']:
LOG.info("MGI Gene id not recognized: %s", mgi_gene_id)
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors - too many. report summary at the end
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol != '' and mgi_gene_id == '':
# LOG.error(
# "Gene label with no MGI identifier for strain %s: %s",
# strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol)
# make a temp id for genes that aren't identified ... err wow.
# tmp_gene_id = '_' + mgi_gene_symbol
# self.id_label_hash[tmp_gene_id.strip()] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mpt_ids are a comma delimited list
# labels with MP terms following in brackets
phenotype_ids = []
if mpt_ids != '':
for lb_mp in mpt_ids.split(r','):
lb_mp = lb_mp.strip()
if lb_mp[-1:] == ']' and lb_mp[-12:-8] == '[MP:':
phenotype_ids.append(lb_mp[-11:-2])
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums != '':
for pm_num in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:' + pm_num.strip()
pubmed_ids.append(pmid)
ref = Reference(graph, pmid,
self.globaltt['journal article'])
ref.addRefToGraph()
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
model.addClassToGraph(mouse_taxon, None)
if research_areas == '':
research_areas = None
else:
research_areas = 'Research Areas: ' + research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
model.addIndividualToGraph( # an inst of mouse??
strain_id, strain_label, strain_type, research_areas)
model.makeLeader(strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in some ontology
model.addClassToGraph(pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(graph, self.name, mgi_allele_id, pid,
self.globaltt['has phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph()
else:
# too chatty here. report aggregate
# LOG.info("Phenotypes and no allele for %s", strain_id)
strain_missing_allele.append(strain_id)
if not self.test_mode and (limit is not None
and reader.line_num > limit):
break
# report misses
if strain_missing_allele:
LOG.info("Phenotypes and no allele for %i strains",
len(strain_missing_allele))
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if variants:
for var in variants:
vl_id = var.strip()
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(vl_id, vl_symbol,
self.globaltt['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_:' + re.sub(r':', '', gene) + '-VL'
vl_symbol = self.id_label_hash[gene] + '<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(vl_id, vl_symbol,
self.globaltt['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = re.sub(r'^_', '', vl) + 'U'
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(vslc_id, vl, None,
self.globaltt['indeterminate'],
self.globaltt['has_variant_part'],
None)
model.addIndividualToGraph(
vslc_id, vslc_label,
self.globaltt['variant single locus complement'])
if vslc_list:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r'_|:', '', gvc_id)
gvc_id = '_:' + gvc_id
gvc_label = '; '.join(self.id_label_hash[v]
for v in vslc_list)
model.addIndividualToGraph(
gvc_id, gvc_label,
self.globaltt['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = re.sub(
r':', '', '-'.join(
(self.globaltt['unspecified_genomic_background'],
s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
bkgd_id = '_:' + bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified (' + s + ')',
self.globaltt['unspecified_genomic_background'],
"A placeholder for the unspecified genetic background for "
+ s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
self.globaltt['unspecified_genomic_background'])
geno.addParts(gvc_id, genotype_id,
self.globaltt['has_variant_part'])
geno.addGenotype(genotype_id, genotype_label)
graph.addTriple(s, self.globaltt['has_genotype'],
genotype_id)
else:
# LOG.debug(
# "Strain %s is not making a proper genotype.", s)
pass
LOG.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
LOG.error('%i symbols given are missing their gene identifiers',
len(genes_with_no_ids))
return
def _process_phenotype_data(self, limit):
"""
NOTE: If a Strain carries more than one mutation,
then each Mutation description,
i.e., the set: (
Mutation Type - Chromosome - Gene Symbol -
Gene Name - Allele Symbol - Allele Name)
will require a separate line.
Note that MMRRC curates phenotypes to alleles,
even though they distribute only one file with the
phenotypes appearing to be associated with a strain.
So, here we process the allele-to-phenotype relationships separately
from the strain-to-allele relationships.
:param limit:
:return:
"""
src_key = 'catalog'
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
fname = '/'.join((self.rawdir, self.files[src_key]['file']))
self.strain_hash = {}
self.id_label_hash = {}
genes_with_no_ids = set()
stem_cell_class = self.globaltt['stem cell']
mouse_taxon = self.globaltt['Mus musculus']
geno = Genotype(graph)
with open(fname, 'r', encoding="utf8") as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='\"')
# This MMRRC catalog data file was generated on YYYY-MM-DD
# insert or check date w/dataset
line = next(reader)
# gen_date = line[-10:]
line = next(reader)
col = self.files['catalog']['columns']
if col != line:
LOG.error(
'%s\nExpected Headers:\t%s\nRecived Headers:\t%s\n',
src_key, col, line)
LOG.info(set(col) - set(line))
line = next(reader)
if line != []:
LOG.warning('Expected third line to be blank. got "%s" instead', line)
for row in reader:
strain_id = row[col.index('STRAIN/STOCK_ID')].strip()
strain_label = row[col.index('STRAIN/STOCK_DESIGNATION')]
# strain_type_symbol = row[col.index('STRAIN_TYPE')]
strain_state = row[col.index('STATE')]
mgi_allele_id = row[col.index('MGI_ALLELE_ACCESSION_ID')].strip()
mgi_allele_symbol = row[col.index('ALLELE_SYMBOL')]
# mgi_allele_name = row[col.index('ALLELE_NAME')]
# mutation_type = row[col.index('MUTATION_TYPE')]
# chrom = row[col.index('CHROMOSOME')]
mgi_gene_id = row[col.index('MGI_GENE_ACCESSION_ID')].strip()
mgi_gene_symbol = row[col.index('GENE_SYMBOL')].strip()
mgi_gene_name = row[col.index('GENE_NAME')]
# sds_url = row[col.index('SDS_URL')]
# accepted_date = row[col.index('ACCEPTED_DATE')]
mpt_ids = row[col.index('MPT_IDS')].strip()
pubmed_nums = row[col.index('PUBMED_IDS')].strip()
research_areas = row[col.index('RESEARCH_AREAS')].strip()
if self.test_mode and (strain_id not in self.test_ids) \
or mgi_gene_name == 'withdrawn':
continue
# strip off stuff after the dash -
# is the holding center important?
# MMRRC:00001-UNC --> MMRRC:00001
strain_id = re.sub(r'-\w+$', '', strain_id)
self.id_label_hash[strain_id] = strain_label
# get the variant or gene to save for later building of
# the genotype
if strain_id not in self.strain_hash:
self.strain_hash[strain_id] = {
'variants': set(), 'genes': set()}
# flag bad ones
if mgi_allele_id[:4] != 'MGI:' and mgi_allele_id != '':
LOG.error("Erroneous MGI allele id: %s", mgi_allele_id)
if mgi_allele_id[:3] == 'MG:':
mgi_allele_id = 'MGI:' + mgi_allele_id[3:]
else:
mgi_allele_id = ''
if mgi_allele_id != '':
self.strain_hash[strain_id]['variants'].add(mgi_allele_id)
self.id_label_hash[mgi_allele_id] = mgi_allele_symbol
# use the following if needing to add the sequence alteration types
# var_type = self.localtt[mutation_type]
# make a sequence alteration for this variant locus,
# and link the variation type to it
# sa_id = '_'+re.sub(r':','',mgi_allele_id)+'SA'
# if self.nobnodes:
# sa_id = ':'+sa_id
# gu.addIndividualToGraph(g, sa_id, None, var_type)
# geno.addSequenceAlterationToVariantLocus(sa_id, mgi_allele_id)
# scrub out any spaces, fix known issues
mgi_gene_id = re.sub(r'\s+', '', mgi_gene_id)
if mgi_gene_id == 'NULL':
mgi_gene_id = ''
elif mgi_gene_id[:7] == 'GeneID:':
mgi_gene_id = 'NCBIGene:' + mgi_gene_id[7:]
if mgi_gene_id != '':
[curie, localid] = mgi_gene_id.split(':')
if curie not in ['MGI', 'NCBIGene']:
LOG.info("MGI Gene id not recognized: %s", mgi_gene_id)
self.strain_hash[strain_id]['genes'].add(mgi_gene_id)
self.id_label_hash[mgi_gene_id] = mgi_gene_symbol
# catch some errors - too many. report summary at the end
# some things have gene labels, but no identifiers - report
if mgi_gene_symbol != '' and mgi_gene_id == '':
# LOG.error(
# "Gene label with no MGI identifier for strain %s: %s",
# strain_id, mgi_gene_symbol)
genes_with_no_ids.add(mgi_gene_symbol)
# make a temp id for genes that aren't identified ... err wow.
# tmp_gene_id = '_' + mgi_gene_symbol
# self.id_label_hash[tmp_gene_id.strip()] = mgi_gene_symbol
# self.strain_hash[strain_id]['genes'].add(tmp_gene_id)
# split apart the mp ids
# ataxia [MP:0001393] ,hypoactivity [MP:0001402] ...
# mpt_ids are a comma delimited list
# labels with MP terms following in brackets
phenotype_ids = []
if mpt_ids != '':
for lb_mp in mpt_ids.split(r','):
lb_mp = lb_mp.strip()
if lb_mp[-1:] == ']' and lb_mp[-12:-8] == '[MP:':
phenotype_ids.append(lb_mp[-11:-2])
# pubmed ids are space delimited
pubmed_ids = []
if pubmed_nums != '':
for pm_num in re.split(r'\s+', pubmed_nums):
pmid = 'PMID:' + pm_num.strip()
pubmed_ids.append(pmid)
ref = Reference(graph, pmid, self.globaltt['journal article'])
ref.addRefToGraph()
# https://www.mmrrc.org/catalog/sds.php?mmrrc_id=00001
# is a good example of 4 genotype parts
model.addClassToGraph(mouse_taxon, None)
if research_areas == '':
research_areas = None
else:
research_areas = 'Research Areas: ' + research_areas
strain_type = mouse_taxon
if strain_state == 'ES':
strain_type = stem_cell_class
model.addIndividualToGraph( # an inst of mouse??
strain_id, strain_label, strain_type, research_areas)
model.makeLeader(strain_id)
# phenotypes are associated with the alleles
for pid in phenotype_ids:
# assume the phenotype label is in some ontology
model.addClassToGraph(pid, None)
if mgi_allele_id is not None and mgi_allele_id != '':
assoc = G2PAssoc(
graph, self.name, mgi_allele_id, pid,
self.globaltt['has phenotype'])
for p in pubmed_ids:
assoc.add_source(p)
assoc.add_association_to_graph()
else:
LOG.info("Phenotypes and no allele for %s", strain_id)
if not self.test_mode and (
limit is not None and reader.line_num > limit):
break
# now that we've collected all of the variant information, build it
# we don't know their zygosities
for s in self.strain_hash:
h = self.strain_hash.get(s)
variants = h['variants']
genes = h['genes']
vl_set = set()
# make variant loci for each gene
if len(variants) > 0:
for var in variants:
vl_id = var.strip()
vl_symbol = self.id_label_hash[vl_id]
geno.addAllele(
vl_id, vl_symbol, self.globaltt['variant_locus'])
vl_set.add(vl_id)
if len(variants) == 1 and len(genes) == 1:
for gene in genes:
geno.addAlleleOfGene(vl_id, gene)
else:
geno.addAllele(vl_id, vl_symbol)
else: # len(vars) == 0
# it's just anonymous variants in some gene
for gene in genes:
vl_id = '_:' + re.sub(r':', '', gene) + '-VL'
vl_symbol = self.id_label_hash[gene]+'<?>'
self.id_label_hash[vl_id] = vl_symbol
geno.addAllele(
vl_id, vl_symbol, self.globaltt['variant_locus'])
geno.addGene(gene, self.id_label_hash[gene])
geno.addAlleleOfGene(vl_id, gene)
vl_set.add(vl_id)
# make the vslcs
vl_list = sorted(vl_set)
vslc_list = []
for vl in vl_list:
# for unknown zygosity
vslc_id = re.sub(r'^_', '', vl)+'U'
vslc_id = re.sub(r':', '', vslc_id)
vslc_id = '_:' + vslc_id
vslc_label = self.id_label_hash[vl] + '/?'
self.id_label_hash[vslc_id] = vslc_label
vslc_list.append(vslc_id)
geno.addPartsToVSLC(
vslc_id, vl, None, self.globaltt['indeterminate'],
self.globaltt['has_variant_part'], None)
model.addIndividualToGraph(
vslc_id, vslc_label,
self.globaltt['variant single locus complement'])
if len(vslc_list) > 0:
if len(vslc_list) > 1:
gvc_id = '-'.join(vslc_list)
gvc_id = re.sub(r'_|:', '', gvc_id)
gvc_id = '_:'+gvc_id
gvc_label = '; '.join(self.id_label_hash[v] for v in vslc_list)
model.addIndividualToGraph(
gvc_id, gvc_label,
self.globaltt['genomic_variation_complement'])
for vslc_id in vslc_list:
geno.addVSLCtoParent(vslc_id, gvc_id)
else:
# the GVC == VSLC, so don't have to make an extra piece
gvc_id = vslc_list.pop()
gvc_label = self.id_label_hash[gvc_id]
genotype_label = gvc_label + ' [n.s.]'
bkgd_id = re.sub(
r':', '', '-'.join((
self.globaltt['unspecified_genomic_background'], s)))
genotype_id = '-'.join((gvc_id, bkgd_id))
bkgd_id = '_:' + bkgd_id
geno.addTaxon(mouse_taxon, bkgd_id)
geno.addGenomicBackground(
bkgd_id, 'unspecified (' + s + ')',
self.globaltt['unspecified_genomic_background'],
"A placeholder for the unspecified genetic background for " + s)
geno.addGenomicBackgroundToGenotype(
bkgd_id, genotype_id,
self.globaltt['unspecified_genomic_background'])
geno.addParts(
gvc_id, genotype_id, self.globaltt['has_variant_part'])
geno.addGenotype(genotype_id, genotype_label)
graph.addTriple(
s, self.globaltt['has_genotype'], genotype_id)
else:
# LOG.debug(
# "Strain %s is not making a proper genotype.", s)
pass
LOG.warning(
"The following gene symbols did not list identifiers: %s",
str(sorted(list(genes_with_no_ids))))
LOG.error(
'%i symbols given are missing their gene identifiers',
len(genes_with_no_ids))
return
