def _process_straininfo(self, limit):
# line_counter = 0 # TODO unused
if self.testMode:
g = self.testgraph
else:
g = self.graph
logger.info("Processing measurements ...")
raw = '/'.join((self.rawdir, self.files['straininfo']['file']))
tax_id = 'NCBITaxon:10090'
gu = GraphUtils(curie_map.get())
with open(raw, 'r') as f:
reader = csv.reader(f, delimiter=',', quotechar='\"')
f.readline() # read the header row; skip
for row in reader:
(strain_name, vendor, stocknum, panel, mpd_strainid,
straintype, n_proj, n_snp_datasets, mpdshortname, url) = row
# C57BL/6J,J,000664,,7,IN,225,17,,http://jaxmice.jax.org/strain/000664.html
# create the strain as an instance of the taxon
if self.testMode and \
'MPD:'+str(mpd_strainid) not in self.test_ids:
continue
strain_id = 'MPD-strain:'+str(mpd_strainid)
gu.addIndividualToGraph(g, strain_id, strain_name, tax_id)
if mpdshortname.strip() != '':
gu.addSynonym(g, strain_id, mpdshortname.strip())
self.idlabel_hash[strain_id] = strain_name
# make it equivalent to the vendor+stock
if stocknum != '':
if vendor == 'J':
jax_id = 'JAX:'+stocknum
gu.addSameIndividual(g, strain_id, jax_id)
elif vendor == 'Rbrc':
# reiken
reiken_id = 'RBRC:'+re.sub(r'RBRC', '', stocknum)
gu.addSameIndividual(g, strain_id, reiken_id)
else:
if url != '':
gu.addXref(g, strain_id, url, True)
if vendor != '':
gu.addXref(
g, strain_id, ':'.join((vendor, stocknum)),
True)
# add the panel information
if panel != '':
desc = panel+' [panel]'
gu.addDescription(g, strain_id, desc)
# TODO make the panels as a resource collection
return
def process_pub_xrefs(self, limit=None):
raw = '/'.join((self.rawdir, self.files['pub_xrefs']['file']))
if self.testMode:
g = self.testgraph
else:
g = self.graph
gu = GraphUtils(curie_map.get())
logger.info("Processing publication xrefs")
line_counter = 0
with open(raw, 'r') as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(wb_ref, xref) = row
# WBPaper00000009 pmid8805<BR>
# WBPaper00000011 doi10.1139/z78-244<BR>
# WBPaper00000012 cgc12<BR>
if self.testMode and wb_ref not in self.test_ids['pub']:
continue
ref_id = 'WormBase:'+wb_ref
xref_id = r = None
xref = re.sub(r'<BR>', '', xref)
xref = xref.strip()
if re.match(r'pmid', xref):
xref_id = 'PMID:'+re.sub(r'pmid\s*', '', xref)
r = Reference(
xref_id, Reference.ref_types['journal_article'])
elif re.search(r'[\(\)\<\>\[\]\s]', xref):
continue
elif re.match(r'doi', xref):
xref_id = 'DOI:'+re.sub(r'doi', '', xref.strip())
r = Reference(xref_id)
elif re.match(r'cgc', xref):
# TODO not sure what to do here with cgc xrefs
continue
else:
# logger.debug("Other xrefs like %s", xref)
continue
if xref_id is not None:
r.addRefToGraph(g)
gu.addSameIndividual(g, ref_id, xref_id)
if not self.testMode \
and limit is not None and line_counter > limit:
break
return
def _process_data(self, raw, limit=None):
"""
This function will process the data files from Coriell.
We make the assumption that any alleles listed are variants
(alternates to w.t.)
Triples: (examples)
:NIGMSrepository a CLO_0000008 #repository
label : NIGMS Human Genetic Cell Repository
foaf:page https://catalog.coriell.org/0/sections/collections/NIGMS/?SsId=8
line_id a CL_0000057, #fibroblast line
derives_from patient_id
part_of :NIGMSrepository
RO:model_of OMIM:disease_id
patient id a foaf:person,
label: "fibroblast from patient 12345 with disease X"
member_of family_id #what is the right thing here?
SIO:race EFO:caucasian #subclass of EFO:0001799
in_taxon NCBITaxon:9606
dc:description Literal(remark)
RO:has_phenotype OMIM:disease_id
GENO:has_genotype genotype_id
family_id a owl:NamedIndividual
foaf:page "https://catalog.coriell.org/0/Sections/BrowseCatalog/FamilyTypeSubDetail.aspx?PgId=402&fam=2104&coll=GM"
genotype_id a intrinsic_genotype
GENO:has_alternate_part allelic_variant_id
we don't necessarily know much about the genotype,
other than the allelic variant. also there's the sex here
pub_id mentions cell_line_id
:param raw:
:param limit:
:return:
"""
logger.info("Processing Data from %s", raw)
gu = GraphUtils(curie_map.get())
if self.testMode: # set the graph to build
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
du = DipperUtil()
gu.loadProperties(g, geno.object_properties, gu.OBJPROP)
gu.loadAllProperties(g)
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
next(filereader, None) # skip the header row
for row in filereader:
if not row:
pass
else:
line_counter += 1
(catalog_id, description, omim_number, sample_type,
cell_line_available, dna_in_stock, dna_ref, gender, age,
race, ethnicity, affected, karyotype, relprob, mutation,
gene, family_id, collection, url, cat_remark, pubmed_ids,
family_member, variant_id, dbsnp_id, species) = row
# example:
# GM00003,HURLER SYNDROME,607014,Fibroblast,Yes,No,,Female,26 YR,Caucasian,,,,
# parent,,,39,NIGMS Human Genetic Cell Repository,
# http://ccr.coriell.org/Sections/Search/Sample_Detail.aspx?Ref=GM00003,
# 46;XX; clinically normal mother of a child with Hurler syndrome; proband not in Repository,,
# 2,,18343,H**o sapiens
if self.testMode and catalog_id not in self.test_lines:
# skip rows not in our test lines, when in test mode
continue
# ########### BUILD REQUIRED VARIABLES ###########
# Make the cell line ID
cell_line_id = 'Coriell:'+catalog_id.strip()
# Map the cell/sample type
cell_type = self._map_cell_type(sample_type)
# Make a cell line label
line_label = \
collection.partition(' ')[0]+'-'+catalog_id.strip()
# Map the repository/collection
repository = self._map_collection(collection)
# patients are uniquely identified by one of:
# dbsnp id (which is == an individual haplotype)
# family id + family member (if present) OR
# probands are usually family member zero
# cell line id
# since some patients have >1 cell line derived from them,
# we must make sure that the genotype is attached to
# the patient, and can be inferred to the cell line
# examples of repeated patients are:
# famid=1159, member=1; fam=152,member=1
# Make the patient ID
# make an anonymous patient
patient_id = '_person'
if self.nobnodes:
patient_id = ':'+patient_id
if family_id != '':
patient_id = \
'-'.join((patient_id, family_id, family_member))
else:
# make an anonymous patient
patient_id = '-'.join((patient_id, catalog_id.strip()))
# properties of the individual patients: sex, family id,
# member/relproband, description descriptions are
# really long and ugly SCREAMING text, so need to clean up
# the control cases are so odd with this labeling scheme;
# but we'll deal with it as-is for now.
short_desc = (description.split(';')[0]).capitalize()
if affected == 'Yes':
affected = 'affected'
elif affected == 'No':
affected = 'unaffected'
gender = gender.lower()
patient_label = ' '.join((affected, gender, relprob))
if relprob == 'proband':
patient_label = \
' '.join(
(patient_label.strip(), 'with', short_desc))
else:
patient_label = \
' '.join(
(patient_label.strip(), 'of proband with',
short_desc))
# ############# BUILD THE CELL LINE #############
# Adding the cell line as a typed individual.
cell_line_reagent_id = 'CLO:0000031'
gu.addIndividualToGraph(
g, cell_line_id, line_label, cell_line_reagent_id)
# add the equivalent id == dna_ref
if dna_ref != '' and dna_ref != catalog_id:
equiv_cell_line = 'Coriell:'+dna_ref
# some of the equivalent ids are not defined
# in the source data; so add them
gu.addIndividualToGraph(
g, equiv_cell_line, None, cell_line_reagent_id)
gu.addSameIndividual(g, cell_line_id, equiv_cell_line)
# Cell line derives from patient
geno.addDerivesFrom(cell_line_id, patient_id)
geno.addDerivesFrom(cell_line_id, cell_type)
# Cell line a member of repository
gu.addMember(g, repository, cell_line_id)
if cat_remark != '':
gu.addDescription(g, cell_line_id, cat_remark)
# Cell age_at_sampling
# TODO add the age nodes when modeled properly in #78
# if (age != ''):
# this would give a BNode that is an instance of Age.
# but i don't know how to connect
# the age node to the cell line? we need to ask @mbrush
# age_id = '_'+re.sub('\s+','_',age)
# gu.addIndividualToGraph(
# g,age_id,age,self.terms['age'])
# gu.addTriple(
# g,age_id,self.properties['has_measurement'],age,
# True)
# ############# BUILD THE PATIENT #############
# Add the patient ID as an individual.
gu.addPerson(g, patient_id, patient_label)
# TODO map relationship to proband as a class
# (what ontology?)
# Add race of patient
# FIXME: Adjust for subcategories based on ethnicity field
# EDIT: There are 743 different entries for ethnicity...
# Too many to map?
# Add ethnicity as literal in addition to the mapped race?
# Adjust the ethnicity txt (if using)
# to initial capitalization to remove ALLCAPS
# TODO race should go into the individual's background
# and abstracted out to the Genotype class punting for now.
# if race != '':
# mapped_race = self._map_race(race)
# if mapped_race is not None:
# gu.addTriple(
# g,patient_id,self.terms['race'],mapped_race)
# gu.addSubclass(
# g,self.terms['ethnic_group'],mapped_race)
# ############# BUILD THE FAMILY #############
# Add triples for family_id, if present.
if family_id != '':
family_comp_id = 'CoriellFamily:'+family_id
family_label = \
' '.join(('Family of proband with', short_desc))
# Add the family ID as a named individual
gu.addIndividualToGraph(
g, family_comp_id, family_label,
geno.genoparts['family'])
# Add the patient as a member of the family
gu.addMemberOf(g, patient_id, family_comp_id)
# ############# BUILD THE GENOTYPE #############
# the important things to pay attention to here are:
# karyotype = chr rearrangements (somatic?)
# mutation = protein-level mutation as a label,
# often from omim
# gene = gene symbol - TODO get id
# variant_id = omim variant ids (; delimited)
# dbsnp_id = snp individual ids = full genotype?
# note GM00633 is a good example of chromosomal variation
# - do we have enough to capture this?
# GM00325 has both abnormal karyotype and variation
# make an assumption that if the taxon is blank,
# that it is human!
if species is None or species == '':
species = 'H**o sapiens'
taxon = self._map_species(species)
# if there's a dbSNP id,
# this is actually the individual's genotype
genotype_id = None
genotype_label = None
if dbsnp_id != '':
genotype_id = 'dbSNPIndividual:'+dbsnp_id.strip()
omim_map = {}
gvc_id = None
# some of the karyotypes are encoded
# with terrible hidden codes. remove them here
# i've seen a <98> character
karyotype = du.remove_control_characters(karyotype)
karyotype_id = None
if karyotype.strip() != '':
karyotype_id = \
'_'+re.sub('MONARCH:', '', self.make_id(karyotype))
if self.nobnodes:
karyotype_id = ':'+karyotype_id
# add karyotype as karyotype_variation_complement
gu.addIndividualToGraph(
g, karyotype_id, karyotype,
geno.genoparts['karyotype_variation_complement'])
# TODO break down the karyotype into parts
# and map into GENO. depends on #77
# place the karyotype in a location(s).
karyo_chrs = \
self._get_affected_chromosomes_from_karyotype(
karyotype)
for c in karyo_chrs:
chr_id = makeChromID(c, taxon, 'CHR')
# add an anonymous sequence feature,
# each located on chr
karyotype_feature_id = '-'.join((karyotype_id, c))
karyotype_feature_label = \
'some karyotype alteration on chr'+str(c)
f = Feature(
karyotype_feature_id, karyotype_feature_label,
geno.genoparts['sequence_alteration'])
f.addFeatureStartLocation(None, chr_id)
f.addFeatureToGraph(g)
f.loadAllProperties(g)
geno.addParts(
karyotype_feature_id, karyotype_id,
geno.object_properties['has_alternate_part'])
if gene != '':
vl = gene+'('+mutation+')'
# fix the variant_id so it's always in the same order
vids = variant_id.split(';')
variant_id = ';'.join(sorted(list(set(vids))))
if karyotype.strip() != '' \
and not self._is_normal_karyotype(karyotype):
mutation = mutation.strip()
gvc_id = karyotype_id
if variant_id != '':
gvc_id = '_' + variant_id.replace(';', '-') + '-' \
+ re.sub(r'\w*:', '', karyotype_id)
if mutation.strip() != '':
gvc_label = '; '.join((vl, karyotype))
else:
gvc_label = karyotype
elif variant_id.strip() != '':
gvc_id = '_' + variant_id.replace(';', '-')
gvc_label = vl
else:
# wildtype?
pass
if gvc_id is not None and gvc_id != karyotype_id \
and self.nobnodes:
gvc_id = ':'+gvc_id
# add the karyotype to the gvc.
# use reference if normal karyotype
karyo_rel = geno.object_properties['has_alternate_part']
if self._is_normal_karyotype(karyotype):
karyo_rel = \
geno.object_properties['has_reference_part']
if karyotype_id is not None \
and not self._is_normal_karyotype(karyotype) \
and gvc_id is not None and karyotype_id != gvc_id:
geno.addParts(karyotype_id, gvc_id, karyo_rel)
if variant_id.strip() != '':
# split the variants & add them as part of the genotype
# we don't necessarily know their zygosity,
# just that they are part of the genotype variant ids
# are from OMIM, so prefix as such we assume that the
# sequence alts will be defined in OMIM not here
# TODO sort the variant_id list, if the omim prefix is
# the same, then assume it's the locus make a hashmap
# of the omim id to variant id list;
# then build the genotype hashmap is also useful for
# removing the "genes" from the list of "phenotypes"
# will hold gene/locus id to variant list
omim_map = {}
locus_num = None
for v in variant_id.split(';'):
# handle omim-style and odd var ids
# like 610661.p.R401X
m = re.match(r'(\d+)\.+(.*)', v.strip())
if m is not None and len(m.groups()) == 2:
(locus_num, var_num) = m.groups()
if locus_num is not None \
and locus_num not in omim_map:
omim_map[locus_num] = [var_num]
else:
omim_map[locus_num] += [var_num]
for o in omim_map:
# gene_id = 'OMIM:' + o # TODO unused
vslc_id = \
'_' + '-'.join(
[o + '.' + a for a in omim_map.get(o)])
if self.nobnodes:
vslc_id = ':'+vslc_id
vslc_label = vl
# we don't really know the zygosity of
# the alleles at all.
# so the vslcs are just a pot of them
gu.addIndividualToGraph(
g, vslc_id, vslc_label,
geno.genoparts[
'variant_single_locus_complement'])
for v in omim_map.get(o):
# this is actually a sequence alt
allele1_id = 'OMIM:'+o+'.'+v
geno.addSequenceAlteration(allele1_id, None)
# assume that the sa -> var_loc -> gene
# is taken care of in OMIM
geno.addPartsToVSLC(
vslc_id, allele1_id, None,
geno.zygosity['indeterminate'],
geno.object_properties[
'has_alternate_part'])
if vslc_id != gvc_id:
geno.addVSLCtoParent(vslc_id, gvc_id)
if affected == 'unaffected':
# let's just say that this person is wildtype
gu.addType(g, patient_id, geno.genoparts['wildtype'])
elif genotype_id is None:
# make an anonymous genotype id
genotype_id = '_geno'+catalog_id.strip()
if self.nobnodes:
genotype_id = ':'+genotype_id
# add the gvc
if gvc_id is not None:
gu.addIndividualToGraph(
g, gvc_id, gvc_label,
geno.genoparts['genomic_variation_complement'])
# add the gvc to the genotype
if genotype_id is not None:
if affected == 'unaffected':
rel = \
geno.object_properties[
'has_reference_part']
else:
rel = \
geno.object_properties[
'has_alternate_part']
geno.addParts(gvc_id, genotype_id, rel)
if karyotype_id is not None \
and self._is_normal_karyotype(karyotype):
if gvc_label is not None and gvc_label != '':
genotype_label = \
'; '.join((gvc_label, karyotype))
else:
genotype_label = karyotype
if genotype_id is None:
genotype_id = karyotype_id
else:
geno.addParts(
karyotype_id, genotype_id,
geno.object_properties[
'has_reference_part'])
else:
genotype_label = gvc_label
# use the catalog id as the background
genotype_label += ' ['+catalog_id.strip()+']'
if genotype_id is not None and gvc_id is not None:
# only add the genotype if it has some parts
geno.addGenotype(
genotype_id, genotype_label,
geno.genoparts['intrinsic_genotype'])
geno.addTaxon(taxon, genotype_id)
# add that the patient has the genotype
# TODO check if the genotype belongs to
# the cell line or to the patient
gu.addTriple(
g, patient_id,
geno.properties['has_genotype'], genotype_id)
else:
geno.addTaxon(taxon, patient_id)
# TODO: Add sex/gender (as part of the karyotype?)
# ############# DEAL WITH THE DISEASES #############
# we associate the disease to the patient
if affected == 'affected':
if omim_number != '':
for d in omim_number.split(';'):
if d is not None and d != '':
# if the omim number is in omim_map,
# then it is a gene not a pheno
if d not in omim_map:
disease_id = 'OMIM:'+d.strip()
# assume the label is taken care of
gu.addClassToGraph(g, disease_id, None)
# add the association:
# the patient has the disease
assoc = G2PAssoc(
self.name, patient_id, disease_id)
assoc.add_association_to_graph(g)
# this line is a model of this disease
# TODO abstract out model into
# it's own association class?
gu.addTriple(
g, cell_line_id,
gu.properties['model_of'],
disease_id)
else:
logger.info(
'removing %s from disease list ' +
'since it is a gene', d)
# ############# ADD PUBLICATIONS #############
if pubmed_ids != '':
for s in pubmed_ids.split(';'):
pubmed_id = 'PMID:'+s.strip()
ref = Reference(pubmed_id)
ref.setType(Reference.ref_types['journal_article'])
ref.addRefToGraph(g)
gu.addTriple(
g, pubmed_id, gu.properties['mentions'],
cell_line_id)
if not self.testMode \
and (limit is not None and line_counter > limit):
break
Assoc(self.name).load_all_properties(g)
return
def _add_variant_cdna_variant_assoc_to_graph(self, row):
"""
Generates relationships between variants and cDNA variants
given a row of data
:param iterable: row of data, see add_variant_info_to_graph()
docstring for expected structure.
Only applicable for structure 2.
:return None
"""
gu = GraphUtils(curie_map.get())
geno = Genotype(self.graph)
is_literal = True
(variant_key, variant_label, amino_acid_variant, amino_acid_position,
transcript_id, transcript_priority, protein_variant_type,
functional_impact, stop_gain_loss, transcript_gene,
protein_variant_source, variant_gene, bp_pos, variant_cdna,
cosmic_id, db_snp_id, genome_pos_start, genome_pos_end, ref_base,
variant_base, primary_transcript_exons,
primary_transcript_variant_sub_types, variant_type, chromosome,
genome_build, build_version, build_date) = row
variant_id = self.make_cgd_id('variant{0}'.format(variant_key))
# Add gene
self._add_variant_gene_relationship(variant_id, variant_gene)
# Transcript reference for nucleotide position
transcript_curie = self._make_transcript_curie(transcript_id)
# Make region IDs
cdna_region_id = ":_{0}Region".format(transcript_curie)
chrom_region_id = ":_{0}{1}Region-{2}-{3}".format(genome_build,
chromosome,
genome_pos_start,
genome_pos_end)
# Add the genome build
genome_label = "Human"
build_id = "UCSC:{0}".format(genome_build)
taxon_id = 'NCBITaxon:9606'
geno.addGenome(taxon_id, genome_label)
geno.addReferenceGenome(build_id, genome_build, taxon_id)
# Add chromosome
chrom_class_id = makeChromID(chromosome, '9606', 'CHR') # the chrom class (generic) id
chrom_instance_id = makeChromID(chromosome, build_id, 'MONARCH')
# first, add the chromosome class (in the taxon)
geno.addChromosomeClass(chromosome, taxon_id, 'Human')
# then, add the chromosome instance (from the given build)
geno.addChromosomeInstance(chromosome, build_id, genome_build, chrom_class_id)
# Add variant coordinates in reference to chromosome
self._add_feature_with_coords(variant_id,genome_pos_start,
genome_pos_end, chrom_instance_id, chrom_region_id)
# Add mutation coordinates in reference to gene
self._add_feature_with_coords(variant_id, bp_pos,
bp_pos, transcript_curie, cdna_region_id)
# Add nucleotide mutation
gu.addTriple(self.graph, variant_id,
geno.properties['reference_nucleotide'],
ref_base, is_literal)
gu.addTriple(self.graph, variant_id,
geno.properties['altered_nucleotide'],
variant_base, is_literal)
"""
Here we update any internal cgd variant IDS with a cosmic ID
or dbSNP ID. Alternatively we could do this using sql rather
than a sparql update which may be safer
"""
# Add SNP xrefs
if cosmic_id is not None:
cosmic_id_list = cosmic_id.split(', ')
cosmic_curie_list = []
for c_id in cosmic_id_list:
cosmic_curie = re.sub(r'COSM(\d+)', r'COSMIC:\1', c_id)
cosmic_curie_list.append(cosmic_curie)
gu.addIndividualToGraph(self.graph, cosmic_curie, c_id,
geno.genoparts['missense_variant'])
# If there are multiple ids set them equivalent to the first
for curie in cosmic_curie_list[1:]:
gu.addSameIndividual(self.graph, cosmic_curie_list[0], curie)
self._replace_entity(self.graph, variant_id, cosmic_curie_list[0], self.bindings)
if db_snp_id is not None:
db_snp_curie = re.sub(r'rs(\d+)', r'dbSNP:\1', db_snp_id)
gu.addIndividualToGraph(self.graph, db_snp_curie, db_snp_id,
geno.genoparts['missense_variant'])
if cosmic_id is None:
self._replace_entity(self.graph, variant_id, db_snp_curie, self.bindings)
else:
cosmic_id_list = cosmic_id.split(', ')
for c_id in cosmic_id_list:
cosmic_curie = re.sub(r'COSM(\d+)', r'COSMIC:\1', c_id)
gu.addSameIndividual(self.graph, cosmic_curie, db_snp_curie)
return
def _add_variant_protein_variant_assoc_to_graph(self, row):
"""
Generates relationships between variants and protein variants
given a row of data
:param iterable: row of data, see add_variant_info_to_graph()
docstring for expected structure
:return None
"""
gu = GraphUtils(curie_map.get())
geno = Genotype(self.graph)
is_missense = False
is_literal = True
(variant_key, variant_label, amino_acid_variant, amino_acid_position,
transcript_id, transcript_priority, protein_variant_type,
functional_impact, stop_gain_loss, transcript_gene,
protein_variant_source) = row[0:11]
variant_id = self.make_cgd_id('variant{0}'.format(variant_key))
transcript_curie = self._make_transcript_curie(transcript_id)
uniprot_curie = self._make_uniprot_polypeptide_curie(transcript_id)
ncbi_protein_curie = self._make_ncbi_polypeptide_curie(transcript_id)
geno.addGenotype(variant_id, variant_label,
geno.genoparts['sequence_alteration'])
# Make fake amino acid sequence in case we
# can't get a CCDS to Uniprot and/or NCBI Protein mapping
aa_seq_id = self.make_cgd_id('transcript{0}'.format(amino_acid_variant))
# Add Transcript:
geno.addTranscript(variant_id, transcript_curie, transcript_id,
geno.genoparts['transcript'])
# Add polypeptide
if ncbi_protein_curie is not None:
geno.addPolypeptide(ncbi_protein_curie,
self.transcript_xrefs['RefSeq'][transcript_id],
transcript_curie)
aa_seq_id = ncbi_protein_curie
if uniprot_curie is not None:
geno.addPolypeptide(uniprot_curie,
self.transcript_xrefs['UniProt'][transcript_id],
transcript_curie)
# Overrides ncbi_protein_curie,
# but we set them as equal individuals below
aa_seq_id = uniprot_curie
if ncbi_protein_curie is not None and uniprot_curie is not None:
gu.addSameIndividual(self.graph, ncbi_protein_curie, uniprot_curie)
else:
aa_seq_id = self.make_cgd_id('transcript{0}'.format(amino_acid_variant))
if protein_variant_type == 'nonsynonymous - missense' \
or re.search(r'missense', variant_label):
is_missense = True
geno.addGenotype(variant_id, variant_label,
geno.genoparts['missense_variant'])
# Get gene ID from gene map
self._add_variant_gene_relationship(variant_id, transcript_gene)
amino_acid_regex = re.compile(r'^p\.([A-Za-z]{1,3})(\d+)([A-Za-z]{1,3})$')
if is_missense:
match = re.match(amino_acid_regex, amino_acid_variant.rstrip())
else:
match = None
if match is not None:
ref_amino_acid = match.group(1)
position = match.group(2)
altered_amino_acid = match.group(3)
else:
logger.debug("Could not parse amino acid information"
" from {0} variant:"
" {1} type: {2}".format(amino_acid_variant,
variant_label,
protein_variant_type))
# Add amino acid change to model
if is_missense is True and match is not None:
gu.addTriple(self.graph, variant_id,
geno.properties['reference_amino_acid'],
ref_amino_acid, is_literal)
gu.addTriple(self.graph, variant_id,
geno.properties['results_in_amino_acid_change'],
altered_amino_acid, is_literal)
aa_region_id = ":_{0}{1}{2}Region".format(position, position, aa_seq_id)
self._add_feature_with_coords(variant_id, position,
position, aa_seq_id, aa_region_id)
return
class OMIA(Source):
"""
This is the parser for the
[Online Mendelian Inheritance in Animals
(OMIA)](http://www.http://omia.angis.org.au),
from which we process inherited disorders, other (single-locus) traits,
and genes in >200 animal species (other than human and mouse and rats).
We generate the omia graph to include the following information:
* genes
* animal taxonomy, and breeds as instances of those taxa
(breeds are akin to "strains" in other taxa)
* animal diseases, along with species-specific subtypes of those diseases
* publications (and their mapping to PMIDs, if available)
* gene-to-phenotype associations (via an anonymous variant-locus
* breed-to-phenotype associations
We make links between OMIA and OMIM in two ways:
1. mappings between OMIA and OMIM are created as OMIA --> hasdbXref OMIM
2. mappings between a breed and OMIA disease are created
to be a model for the mapped OMIM disease,
IF AND ONLY IF it is a 1:1 mapping.
there are some 1:many mappings,
and these often happen if the OMIM item is a gene.
Because many of these species are not covered in
the PANTHER orthology datafiles, we also pull any orthology
relationships from the gene_group files from NCBI.
"""
files = {
'data': {
'file': 'omia.xml.gz',
'url': 'http://omia.angis.org.au/dumps/omia.xml.gz'},
}
def __init__(self):
Source.__init__(self, 'omia')
self.load_bindings()
self.dataset = Dataset(
'omia', 'Online Mendelian Inheritance in Animals',
'http://omia.angis.org.au', None, None,
'http://sydney.edu.au/disclaimer.shtml')
self.id_hash = {
'article': {},
'phene': {},
'breed': {},
'taxon': {},
'gene': {}
}
self.label_hash = {}
self.gu = GraphUtils(curie_map.get())
# used to store the omia to omim phene mappings
self.omia_omim_map = {}
# used to store the unique genes that have phenes
# (for fetching orthology)
self.annotated_genes = set()
self.test_ids = {
'disease': [
'OMIA:001702', 'OMIA:001867', 'OMIA:000478', 'OMIA:000201',
'OMIA:000810', 'OMIA:001400'],
'gene': [
492297, 434, 492296, 3430235, 200685834, 394659996, 200685845,
28713538, 291822383],
'taxon': [9691, 9685, 9606, 9615, 9913, 93934, 37029, 9627, 9825],
# to be filled in during parsing of breed table
# for lookup by breed-associations
'breed': []
}
# to store a map of omia ids and any molecular info
# to write a report for curation
self.stored_omia_mol_gen = {}
self.g = self.graph
self.geno = Genotype(self.g)
return
def fetch(self, is_dl_forced=False):
"""
:param is_dl_forced:
:return:
"""
self.get_files(is_dl_forced)
ncbi = NCBIGene()
# ncbi.fetch()
gene_group = ncbi.files['gene_group']
self.fetch_from_url(
gene_group['url'], '/'.join((ncbi.rawdir, gene_group['file'])),
False)
return
def parse(self, limit=None):
# names of tables to iterate - probably don't need all these:
# Article_Breed, Article_Keyword, Article_Gene, Article_Keyword,
# Article_People, Article_Phene, Articles, Breed, Breed_Phene,
# Genes_gb, Group_Categories, Group_MPO, Inherit_Type, Keywords,
# Landmark, Lida_Links, OMIA_Group, OMIA_author, Omim_Xref, People,
# Phene, Phene_Gene, Publishers, Resources, Species_gb, Synonyms
self.scrub()
if limit is not None:
logger.info("Only parsing first %d rows", limit)
logger.info("Parsing files...")
if self.testOnly:
self.testMode = True
if self.testMode:
self.g = self.testgraph
else:
self.g = self.graph
self.geno = Genotype(self.g)
# we do three passes through the file
# first process species (two others reference this one)
self.process_species(limit)
# then, process the breeds, genes, articles, and other static stuff
self.process_classes(limit)
# next process the association data
self.process_associations(limit)
# process the vertebrate orthology for genes
# that are annotated with phenotypes
ncbi = NCBIGene()
ncbi.add_orthologs_by_gene_group(self.g, self.annotated_genes)
self.load_core_bindings()
self.load_bindings()
logger.info("Done parsing.")
self.write_molgen_report()
return
def scrub(self):
"""
The XML file seems to have mixed-encoding;
we scrub out the control characters
from the file for processing.
:return:
"""
logger.info(
"Scrubbing out the nasty characters that break our parser.")
myfile = '/'.join((self.rawdir, self.files['data']['file']))
tmpfile = '/'.join((self.rawdir, self.files['data']['file']+'.tmp.gz'))
t = gzip.open(tmpfile, 'wb')
du = DipperUtil()
with gzip.open(myfile, 'rb') as f:
filereader = io.TextIOWrapper(f, newline="")
for l in filereader:
l = du.remove_control_characters(l) + '\n'
t.write(l.encode('utf-8'))
t.close()
# move the temp file
logger.info("Replacing the original data with the scrubbed file.")
shutil.move(tmpfile, myfile)
return
# ###################### XML LOOPING FUNCTIONS ##################
def process_species(self, limit):
"""
Loop through the xml file and process the species.
We add elements to the graph, and store the
id-to-label in the label_hash dict.
:param limit:
:return:
"""
myfile = '/'.join((self.rawdir, self.files['data']['file']))
f = gzip.open(myfile, 'rb')
filereader = io.TextIOWrapper(f, newline="")
filereader.readline() # remove the xml declaration line
for event, elem in ET.iterparse(filereader):
# Species ids are == genbank species ids!
self.process_xml_table(
elem, 'Species_gb', self._process_species_table_row, limit)
f.close()
return
def process_classes(self, limit):
"""
Loop through the xml file and process the articles,
breed, genes, phenes, and phenotype-grouping classes.
We add elements to the graph,
and store the id-to-label in the label_hash dict,
along with the internal key-to-external id in the id_hash dict.
The latter are referenced in the association processing functions.
:param limit:
:return:
"""
myfile = '/'.join((self.rawdir, self.files['data']['file']))
f = gzip.open(myfile, 'rb')
filereader = io.TextIOWrapper(f, newline="")
filereader.readline() # remove the xml declaration line
parser = ET.XMLParser(encoding='utf-8')
for event, elem in ET.iterparse(filereader, parser=parser):
self.process_xml_table(
elem, 'Articles', self._process_article_row, limit)
self.process_xml_table(
elem, 'Breed', self._process_breed_row, limit)
self.process_xml_table(
elem, 'Genes_gb', self._process_gene_row, limit)
self.process_xml_table(
elem, 'OMIA_Group', self._process_omia_group_row, limit)
self.process_xml_table(
elem, 'Phene', self._process_phene_row, limit)
self.process_xml_table(
elem, 'Omim_Xref', self._process_omia_omim_map, limit)
f.close()
# post-process the omia-omim associations to filter out the genes
# (keep only phenotypes/diseases)
self.clean_up_omim_genes()
return
def process_associations(self, limit):
"""
Loop through the xml file and process the article-breed, article-phene,
breed-phene, phene-gene associations, and the external links to LIDA.
:param limit:
:return:
"""
myfile = '/'.join((self.rawdir, self.files['data']['file']))
f = gzip.open(myfile, 'rb')
filereader = io.TextIOWrapper(f, newline="")
filereader.readline() # remove the xml declaration line
for event, elem in ET.iterparse(filereader):
self.process_xml_table(
elem, 'Article_Breed', self._process_article_breed_row, limit)
self.process_xml_table(
elem, 'Article_Phene', self._process_article_phene_row, limit)
self.process_xml_table(
elem, 'Breed_Phene', self._process_breed_phene_row, limit)
self.process_xml_table(
elem, 'Lida_Links', self._process_lida_links_row, limit)
self.process_xml_table(
elem, 'Phene_Gene', self._process_phene_gene_row, limit)
self.process_xml_table(
elem, 'Group_MPO', self._process_group_mpo_row, limit)
f.close()
return
# ############ INDIVIDUAL TABLE-LEVEL PROCESSING FUNCTIONS ################
def _process_species_table_row(self, row):
# gb_species_id, sci_name, com_name, added_by, date_modified
tax_id = 'NCBITaxon:'+str(row['gb_species_id'])
sci_name = row['sci_name']
com_name = row['com_name']
if self.testMode and \
(int(row['gb_species_id']) not in self.test_ids['taxon']):
return
self.gu.addClassToGraph(self.g, tax_id, sci_name)
if com_name != '':
self.gu.addSynonym(self.g, tax_id, com_name)
self.label_hash[tax_id] = com_name # for lookup later
else:
self.label_hash[tax_id] = sci_name
return
def _process_breed_row(self, row):
# in test mode, keep all breeds of our test species
if self.testMode and \
(int(row['gb_species_id']) not in self.test_ids['taxon']):
return
# save the breed keys in the test_ids for later processing
self.test_ids['breed'] += [int(row['breed_id'])]
breed_id = self.make_breed_id(row['breed_id'])
self.id_hash['breed'][row['breed_id']] = breed_id
tax_id = 'NCBITaxon:'+str(row['gb_species_id'])
breed_label = row['breed_name']
species_label = self.label_hash.get(tax_id)
if species_label is not None:
breed_label = breed_label + ' ('+species_label+')'
self.gu.addIndividualToGraph(self.g, breed_id, breed_label, tax_id)
self.label_hash[breed_id] = breed_label
return
def _process_phene_row(self, row):
phenotype_id = None
sp_phene_label = row['phene_name']
if sp_phene_label == '':
sp_phene_label = None
if 'omia_id' not in row:
logger.info("omia_id not present for %s", row['phene_id'])
omia_id = self._make_internal_id('phene', phenotype_id)
else:
omia_id = 'OMIA:'+str(row['omia_id'])
if self.testMode and not\
(int(row['gb_species_id']) in self.test_ids['taxon'] and
omia_id in self.test_ids['disease']):
return
# add to internal hash store for later lookup
self.id_hash['phene'][row['phene_id']] = omia_id
descr = row['summary']
if descr == '':
descr = None
# omia label
omia_label = self.label_hash.get(omia_id)
# add the species-specific subclass (TODO please review this choice)
gb_species_id = row['gb_species_id']
if gb_species_id != '':
sp_phene_id = '-'.join((omia_id, gb_species_id))
else:
logger.error(
"No species supplied in species-specific phene table for %s",
omia_id)
return
species_id = 'NCBITaxon:'+str(gb_species_id)
# use this instead
species_label = self.label_hash.get('NCBITaxon:'+gb_species_id)
if sp_phene_label is None and \
omia_label is not None and species_label is not None:
sp_phene_label = ' '.join((omia_label, 'in', species_label))
self.gu.addClassToGraph(
self.g, sp_phene_id, sp_phene_label, omia_id, descr)
# add to internal hash store for later lookup
self.id_hash['phene'][row['phene_id']] = sp_phene_id
self.label_hash[sp_phene_id] = sp_phene_label
# add each of the following descriptions,
# if they are populated, with a tag at the end.
for item in [
'clin_feat', 'history', 'pathology', 'mol_gen', 'control']:
if row[item] is not None and row[item] != '':
self.gu.addDescription(
self.g, sp_phene_id, row[item] + ' ['+item+']')
# if row['symbol'] is not None: # species-specific
# CHECK ME - sometimes spaces or gene labels
# gu.addSynonym(g, sp_phene, row['symbol'])
self.gu.addOWLPropertyClassRestriction(
self.g, sp_phene_id, self.gu.object_properties['in_taxon'],
species_id)
# add inheritance as an association
inheritance_id = self._map_inheritance_term_id(row['inherit'])
if inheritance_id is not None:
assoc = DispositionAssoc(self.name, sp_phene_id, inheritance_id)
assoc.add_association_to_graph(self.g)
if row['characterised'] == 'Yes':
self.stored_omia_mol_gen[omia_id] = {
'mol_gen': row['mol_gen'],
'map_info': row['map_info'],
'species': row['gb_species_id']}
return
def write_molgen_report(self):
import csv
logger.info("Writing G2P report for OMIA")
f = '/'.join((self.outdir, 'omia_molgen_report.txt'))
with open(f, 'w', newline='\n') as csvfile:
writer = csv.writer(csvfile, delimiter='\t')
# write header
h = ['omia_id', 'molecular_description', 'mapping_info', 'species']
writer.writerow(h)
for phene in self.stored_omia_mol_gen:
writer.writerow((str(phene),
self.stored_omia_mol_gen[phene]['mol_gen'],
self.stored_omia_mol_gen[phene]['map_info'],
self.stored_omia_mol_gen[phene]['species']))
logger.info(
"Wrote %d potential G2P descriptions for curation to %s",
len(self.stored_omia_mol_gen), f)
return
def _process_article_row(self, row):
# don't bother in test mode
if self.testMode:
return
iarticle_id = self._make_internal_id('article', row['article_id'])
self.id_hash['article'][row['article_id']] = iarticle_id
rtype = None
if row['journal'] != '':
rtype = Reference.ref_types['journal_article']
r = Reference(iarticle_id, rtype)
if row['title'] is not None:
r.setTitle(row['title'].strip())
if row['year'] is not None:
r.setYear(row['year'])
r.addRefToGraph(self.g)
if row['pubmed_id'] is not None:
pmid = 'PMID:'+str(row['pubmed_id'])
self.id_hash['article'][row['article_id']] = pmid
self.gu.addSameIndividual(self.g, iarticle_id, pmid)
self.gu.addComment(self.g, pmid, iarticle_id)
return
def _process_omia_group_row(self, row):
omia_id = 'OMIA:'+row['omia_id']
if self.testMode and omia_id not in self.test_ids['disease']:
return
group_name = row['group_name']
group_summary = row['group_summary']
disease_id = None
group_category = row.get('group_category')
disease_id = \
self.map_omia_group_category_to_ontology_id(group_category)
if disease_id is not None:
self.gu.addClassToGraph(self.g, disease_id, None)
if disease_id == 'MP:0008762': # embryonic lethal
# add this as a phenotype association
# add embryonic onset
assoc = D2PAssoc(self.name, omia_id, disease_id)
assoc.add_association_to_graph(self.g)
disease_id = None
else:
logger.info(
"No disease superclass defined for %s: %s",
omia_id, group_name)
# default to general disease FIXME this may not be desired
disease_id = 'DOID:4'
if group_summary == '':
group_summary = None
if group_name == '':
group_name = None
self.gu.addClassToGraph(
self.g, omia_id, group_name, disease_id, group_summary)
self.label_hash[omia_id] = group_name
return
def _process_gene_row(self, row):
if self.testMode and row['gene_id'] not in self.test_ids['gene']:
return
gene_id = 'NCBIGene:'+str(row['gene_id'])
self.id_hash['gene'][row['gene_id']] = gene_id
gene_label = row['symbol']
self.label_hash[gene_id] = gene_label
tax_id = 'NCBITaxon:'+str(row['gb_species_id'])
gene_type_id = NCBIGene.map_type_of_gene(row['gene_type'])
self.gu.addClassToGraph(self.g, gene_id, gene_label, gene_type_id)
self.geno.addTaxon(tax_id, gene_id)
return
def _process_article_breed_row(self, row):
# article_id, breed_id, added_by
# don't bother putting these into the test... too many!
# and int(row['breed_id']) not in self.test_ids['breed']:
if self.testMode:
return
article_id = self.id_hash['article'].get(row['article_id'])
breed_id = self.id_hash['breed'].get(row['breed_id'])
# there's some missing data (article=6038). in that case skip
if article_id is not None:
self.gu.addTriple(
self.g, article_id, self.gu.object_properties['is_about'],
breed_id)
else:
logger.warning("Missing article key %s", str(row['article_id']))
return
def _process_article_phene_row(self, row):
"""
Linking articles to species-specific phenes.
:param row:
:return:
"""
# article_id, phene_id, added_by
# look up the article in the hashmap
phenotype_id = self.id_hash['phene'].get(row['phene_id'])
article_id = self.id_hash['article'].get(row['article_id'])
omia_id = self._get_omia_id_from_phene_id(phenotype_id)
if self.testMode and omia_id not in self.test_ids['disease'] \
or phenotype_id is None or article_id is None:
return
# make a triple, where the article is about the phenotype
self.gu.addTriple(
self.g, article_id,
self.gu.object_properties['is_about'], phenotype_id)
return
def _process_breed_phene_row(self, row):
# Linking disorders/characteristic to breeds
# breed_id, phene_id, added_by
breed_id = self.id_hash['breed'].get(row['breed_id'])
phene_id = self.id_hash['phene'].get(row['phene_id'])
# get the omia id
omia_id = self._get_omia_id_from_phene_id(phene_id)
if (self.testMode and not (
omia_id in self.test_ids['disease'] and
int(row['breed_id']) in self.test_ids['breed']) or
breed_id is None or phene_id is None):
return
# FIXME we want a different relationship here
assoc = G2PAssoc(
self.name, breed_id, phene_id,
self.gu.object_properties['has_phenotype'])
assoc.add_association_to_graph(self.g)
# add that the breed is a model of the human disease
# use the omia-omim mappings for this
# we assume that we have already scrubbed out the genes
# from the omim list, so we can make the model associations here
omim_ids = self.omia_omim_map.get(omia_id)
eco_id = "ECO:0000214" # biological aspect of descendant evidence
if omim_ids is not None and len(omim_ids) > 0:
if len(omim_ids) > 1:
logger.info(
"There's 1:many omia:omim mapping: %s, %s",
omia_id, str(omim_ids))
for i in omim_ids:
assoc = G2PAssoc(
self.name, breed_id, i,
self.gu.object_properties['model_of'])
assoc.add_evidence(eco_id)
assoc.add_association_to_graph(self.g)
aid = assoc.get_association_id()
breed_label = self.label_hash.get(breed_id)
if breed_label is None:
breed_label = "this breed"
m = re.search(r'\((.*)\)', breed_label)
if m:
sp_label = m.group(1)
else:
sp_label = ''
phene_label = self.label_hash.get(phene_id)
if phene_label is None:
phene_label = "phenotype"
elif phene_label.endswith(sp_label):
# some of the labels we made already include the species;
# remove it to make a cleaner desc
phene_label = re.sub(r' in '+sp_label, '', phene_label)
desc = ' '.join(
("High incidence of", phene_label, "in", breed_label,
"suggests it to be a model of disease", i + "."))
self.gu.addDescription(self.g, aid, desc)
return
def _process_lida_links_row(self, row):
# lidaurl, omia_id, added_by
omia_id = 'OMIA:'+row['omia_id']
lidaurl = row['lidaurl']
if self.testMode and omia_id not in self.test_ids['disease']:
return
self.gu.addXref(self.g, omia_id, lidaurl, True)
return
def _process_phene_gene_row(self, row):
gene_id = self.id_hash['gene'].get(row['gene_id'])
phene_id = self.id_hash['phene'].get(row['phene_id'])
omia_id = self._get_omia_id_from_phene_id(phene_id)
if self.testMode and not (
omia_id in self.test_ids['disease'] and
row['gene_id'] in self.test_ids['gene']) or\
gene_id is None or phene_id is None:
return
# occasionally some phenes are missing! (ex: 406)
if phene_id is None:
logger.warning("Phene id %s is missing", str(row['phene_id']))
return
gene_label = self.label_hash[gene_id]
# some variant of gene_id has phenotype d
vl = '_'+re.sub(r'NCBIGene:', '', str(gene_id)) + 'VL'
if self.nobnodes:
vl = ':'+vl
self.geno.addAllele(vl, 'some variant of ' + gene_label)
self.geno.addAlleleOfGene(vl, gene_id)
assoc = G2PAssoc(self.name, vl, phene_id)
assoc.add_association_to_graph(self.g)
# add the gene id to the set of annotated genes
# for later lookup by orthology
self.annotated_genes.add(gene_id)
return
def _process_omia_omim_map(self, row):
"""
Links OMIA groups to OMIM equivalents.
:param row:
:return:
"""
# omia_id, omim_id, added_by
omia_id = 'OMIA:'+row['omia_id']
omim_id = 'OMIM:'+row['omim_id']
# also store this for use when we say that a given animal is
# a model of a disease
if omia_id not in self.omia_omim_map:
self.omia_omim_map[omia_id] = set()
self.omia_omim_map[omia_id].add(omim_id)
if self.testMode and omia_id not in self.test_ids['disease']:
return
self.gu.addXref(self.g, omia_id, omim_id)
return
def map_omia_group_category_to_ontology_id(self, category_num):
"""
Using the category number in the OMIA_groups table,
map them to a disease id.
This may be superceeded by other MONDO methods.
Platelet disorders will be more specific once
https://github.com/obophenotype/human-disease-ontology/issues/46
is fulfilled.
:param category_num:
:return:
"""
category_map = {
1: 'DOID:0014667', # Inborn error of metabolism
2: 'MESH:D004392', # Dwarfism
3: 'DOID:1682', # congenital heart disease
4: 'DOID:74', # blood system disease
5: 'DOID:3211', # lysosomal storage disease
6: 'DOID:16', # integumentary system disease
# --> retinal degeneration ==> OMIA:000830
7: 'DOID:8466', # progressive retinal atrophy
8: 'DOID:0050572', # Cone–rod dystrophy
9: 'MESH:C536122', # stationary night blindness
10: 'Orphanet:98553', # developmental retinal disorder
11: 'DOID:5679', # retinal disorder
12: 'Orphanet:90771', # Disorder of Sex Development
# - what to do about this one?
13: 'MP:0008762', # embryonic lethal
# - not sure what to do with this
14: None, # blood group
# FIXME make me more specific
15: 'DOID:2218', # intrinsic platelet disorder
# FIXME make me more specific
16: 'DOID:2218', # extrinsic platelet disorder
17: None # transgenic ???
}
disease_id = None
if category_num is not None and int(category_num) in category_map:
disease_id = category_map.get(int(category_num))
logger.info(
"Found %s for category %s", str(disease_id), str(category_num))
else:
logger.info(
"There's a group category I don't know anything about: %s",
str(category_num))
return disease_id
def _process_group_mpo_row(self, row):
"""
Make OMIA to MP associations
:param row:
:return:
"""
omia_id = 'OMIA:'+row['omia_id']
mpo_num = int(row['MPO_no'])
mpo_id = 'MP:'+str(mpo_num).zfill(7)
assoc = D2PAssoc(self.name, omia_id, mpo_id)
assoc.add_association_to_graph(self.g)
return
def clean_up_omim_genes(self):
omim = OMIM()
# get all the omim ids
allomimids = set()
for omia in self.omia_omim_map:
allomimids.update(self.omia_omim_map[omia])
entries_that_are_phenotypes = omim.process_entries(
list(allomimids), filter_keep_phenotype_entry_ids, None, None)
logger.info(
"Filtered out %d/%d entries that are genes or features",
len(allomimids)-len(entries_that_are_phenotypes), len(allomimids))
# now iterate again and remove those non-phenotype ids
removed_count = 0
for omia in self.omia_omim_map:
ids = self.omia_omim_map[omia]
cleanids = set()
for i in ids:
if i in entries_that_are_phenotypes:
cleanids.add(i)
else:
removed_count += 1 # keep track of how many we've removed
self.omia_omim_map[omia] = cleanids
logger.info(
"Removed %d omim ids from the omia-to-omim map", removed_count)
return
def _make_internal_id(self, prefix, key):
iid = '_'+''.join(('omia', prefix, 'key', str(key)))
if self.nobnodes:
iid = ':'+iid
return iid
def make_breed_id(self, key):
breed_id = 'OMIA-breed:'+str(key)
return breed_id
@staticmethod
def _get_omia_id_from_phene_id(phene_id):
omia_id = None
if phene_id is not None:
m = re.match(r'OMIA:\d+', str(phene_id))
if m:
omia_id = m.group(0)
return omia_id
@staticmethod
def _map_inheritance_term_id(inheritance_symbol):
inherit_map = {
'A': None, # Autosomal
'ACD': 'GENO:0000143', # Autosomal co-dominant
'ADV': None, # autosomal dominant with variable expressivity
'AID': 'GENO:0000259', # autosomal incompletely dominant
'ASD': 'GENO:0000145', # autosomal semi-dominant
# autosomal recessive, semi-lethal
# using generic autosomal recessive
'ASL': 'GENO:0000150',
'D': 'GENO:0000147', # autosomal dominant
'M': None, # multifactorial
'MAT': None, # Maternal
# probably autosomal recessive
# using generic autosomal recessive
'PR': 'GENO:0000150',
'R': 'GENO:0000150', # Autosomal Recessive
# Recessive Embryonic Lethal
# using plain recessive
'REL': 'GENO:0000148',
# Autosomal Recessive Lethal
# using plain autosomal recessive
'RL': 'GENO:0000150',
'S': 'GENO:0000146', # Sex-linked <--using allosomal dominant
'SLi': None, # Sex-limited
'UD': 'GENO:0000144', # Dominant
'X': None, # x-linked # HP:0001417 ?
# X-linked Dominant <-- temp using allosomal dominant FIXME
'XLD': 'GENO:0000146',
# X-linked Recessive <-- temp using allosomal recessive FIXME
'XLR': 'GENO:0000149',
'Y': None, # Y-linked
'Z': None, # Z-linked
# Z-linked recessive <-- temp using allosomal recessive FIXME
'ZR': 'GENO:0000149',
'999': None, # Z-linked incompletely dominant
}
inheritance_id = inherit_map.get(inheritance_symbol)
if inheritance_id is None and inheritance_symbol is not None:
logger.warning(
"No inheritance id is mapped for %s", inheritance_symbol)
return inheritance_id
def getTestSuite(self):
import unittest
from tests.test_omia import OMIATestCase
test_suite = unittest.TestLoader().loadTestsFromTestCase(OMIATestCase)
return test_suite
def _get_gene_info(self, limit):
"""
Currently loops through the gene_info file and
creates the genes as classes, typed with SO. It will add their label,
any alternate labels as synonyms, alternate ids as equivlaent classes.
HPRDs get added as protein products.
The chromosome and chr band get added as blank node regions,
and the gene is faldo:located
on the chr band.
:param limit:
:return:
"""
gu = GraphUtils(curie_map.get())
if self.testMode:
g = self.testgraph
else:
g = self.graph
geno = Genotype(g)
# not unzipping the file
logger.info("Processing Gene records")
line_counter = 0
myfile = '/'.join((self.rawdir, self.files['gene_info']['file']))
logger.info("FILE: %s", myfile)
# Add taxa and genome classes for those in our filter
for tax_num in self.tax_ids:
tax_id = ':'.join(('NCBITaxon', str(tax_num)))
# tax label can get added elsewhere
geno.addGenome(tax_id, str(tax_num))
# label added elsewhere
gu.addClassToGraph(g, tax_id, None)
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match(r'^#', line):
continue
(tax_num, gene_num, symbol, locustag, synonyms, xrefs, chrom,
map_loc, desc, gtype, authority_symbol, name,
nomenclature_status, other_designations,
modification_date) = line.split('\t')
# ##set filter=None in init if you don't want to have a filter
# if self.filter is not None:
# if ((self.filter == 'taxids' and \
# (int(tax_num) not in self.tax_ids))
# or (self.filter == 'geneids' and \
# (int(gene_num) not in self.gene_ids))):
# continue
# #### end filter
if self.testMode and int(gene_num) not in self.gene_ids:
continue
if not self.testMode and int(tax_num) not in self.tax_ids:
continue
line_counter += 1
gene_id = ':'.join(('NCBIGene', gene_num))
tax_id = ':'.join(('NCBITaxon', tax_num))
gene_type_id = self.map_type_of_gene(gtype.strip())
if symbol == 'NEWENTRY':
label = None
else:
label = symbol
# sequence feature, not a gene
if gene_type_id == 'SO:0000110':
self.class_or_indiv[gene_id] = 'I'
else:
self.class_or_indiv[gene_id] = 'C'
if not self.testMode and \
limit is not None and line_counter > limit:
continue
if self.class_or_indiv[gene_id] == 'C':
gu.addClassToGraph(g, gene_id, label, gene_type_id, desc)
# NCBI will be the default leader,
# so we will not add the leader designation here.
else:
gu.addIndividualToGraph(
g, gene_id, label, gene_type_id, desc)
# in this case, they aren't genes.
# so we want someone else to be the leader.
if name != '-':
gu.addSynonym(g, gene_id, name)
if synonyms.strip() != '-':
for s in synonyms.split('|'):
gu.addSynonym(
g, gene_id, s.strip(),
Assoc.annotation_properties['hasRelatedSynonym'])
if other_designations.strip() != '-':
for s in other_designations.split('|'):
gu.addSynonym(
g, gene_id, s.strip(),
Assoc.annotation_properties['hasRelatedSynonym'])
# deal with the xrefs
# MIM:614444|HGNC:HGNC:16851|Ensembl:ENSG00000136828|HPRD:11479|Vega:OTTHUMG00000020696
if xrefs.strip() != '-':
for r in xrefs.strip().split('|'):
fixedr = self._cleanup_id(r)
if fixedr is not None and fixedr.strip() != '':
if re.match(r'HPRD', fixedr):
# proteins are not == genes.
gu.addTriple(
g, gene_id,
self.properties[
'has_gene_product'], fixedr)
else:
# skip some of these for now
if fixedr.split(':')[0] not in [
'Vega', 'IMGT/GENE-DB']:
if self.class_or_indiv.get(gene_id) == 'C':
gu.addEquivalentClass(
g, gene_id, fixedr)
else:
gu.addSameIndividual(
g, gene_id, fixedr)
# edge cases of id | symbol | chr | map_loc:
# 263 AMD1P2 X|Y with Xq28 and Yq12
# 438 ASMT X|Y with Xp22.3 or Yp11.3 # in PAR
# no idea why there's two bands listed - possibly 2 assemblies
# 419 ART3 4 with 4q21.1|4p15.1-p14
# 28227 PPP2R3B X|Y Xp22.33; Yp11.3 # in PAR
# this is of "unknown" type == susceptibility
# 619538 OMS 10|19|3 10q26.3;19q13.42-q13.43;3p25.3
# unlocated scaffold
# 101928066 LOC101928066 1|Un -\
# mouse --> 2C3
# 11435 Chrna1 2 2 C3|2 43.76 cM
# mouse --> 11B1.1
# 11548 Adra1b 11 11 B1.1|11 25.81 cM
# 11717 Ampd3 7 7 57.85 cM|7 E2-E3 # mouse
# 14421 B4galnt1 10 10 D3|10 74.5 cM # mouse
# 323212 wu:fb92e12 19|20 - # fish
# 323368 ints10 6|18 - # fish
# 323666 wu:fc06e02 11|23 - # fish
# feel that the chr placement can't be trusted in this table
# when there is > 1 listed
# with the exception of human X|Y,
# we will only take those that align to one chr
# FIXME remove the chr mapping below
# when we pull in the genomic coords
if str(chrom) != '-' and str(chrom) != '':
if re.search(r'\|', str(chrom)) and \
str(chrom) not in ['X|Y', 'X; Y']:
# means that there's uncertainty in the mapping.
# so skip it
# TODO we'll need to figure out how to deal with
# >1 loc mapping
logger.info(
'%s is non-uniquely mapped to %s.' +
' Skipping for now.',
gene_id, str(chr))
continue
# X|Y Xp22.33;Yp11.3
# if(not re.match(
# r'(\d+|(MT)|[XY]|(Un)$',str(chr).strip())):
# print('odd chr=',str(chr))
if str(chrom) == 'X; Y':
chrom = 'X|Y' # rewrite the PAR regions for processing
# do this in a loop to allow PAR regions like X|Y
for c in re.split(r'\|', str(chrom)):
# assume that the chromosome label is added elsewhere
geno.addChromosomeClass(c, tax_id, None)
mychrom = makeChromID(c, tax_num, 'CHR')
# temporarily use taxnum for the disambiguating label
mychrom_syn = makeChromLabel(c, tax_num)
gu.addSynonym(g, mychrom, mychrom_syn)
band_match = re.match(
r'[0-9A-Z]+[pq](\d+)?(\.\d+)?$', map_loc)
if band_match is not None and \
len(band_match.groups()) > 0:
# if tax_num != '9606':
# continue
# this matches the regular kind of chrs,
# so make that kind of band
# not sure why this matches?
# chrX|Y or 10090chr12|Un"
# TODO we probably need a different regex
# per organism
# the maploc_id already has the numeric chromosome
# in it, strip it first
bid = re.sub(r'^'+c, '', map_loc)
# the generic location (no coordinates)
maploc_id = makeChromID(c+bid, tax_num, 'CHR')
# print(map_loc,'-->',bid,'-->',maploc_id)
# Assume it's type will be added elsewhere
band = Feature(maploc_id, None, None)
band.addFeatureToGraph(g)
# add the band as the containing feature
gu.addTriple(
g, gene_id,
Feature.object_properties['is_subsequence_of'],
maploc_id)
else:
# TODO handle these cases: examples are:
# 15q11-q22,Xp21.2-p11.23,15q22-qter,10q11.1-q24,
# 12p13.3-p13.2|12p13-p12,1p13.3|1p21.3-p13.1,
# 12cen-q21,22q13.3|22q13.3
logger.debug(
'not regular band pattern for %s: %s',
gene_id, map_loc)
# add the gene as a subsequence of the chromosome
gu.addTriple(
g, gene_id,
Feature.object_properties['is_subsequence_of'],
mychrom)
geno.addTaxon(tax_id, gene_id)
gu.loadProperties(g, Feature.object_properties, gu.OBJPROP)
gu.loadProperties(g, Feature.data_properties, gu.DATAPROP)
gu.loadProperties(g, Genotype.object_properties, gu.OBJPROP)
gu.loadAllProperties(g)
return
class UCSCBands(Source):
"""
This will take the UCSC defintions of cytogenic bands and create the nested
structures to enable overlap and containment queries. We use
```Monochrom.py``` to create the OWL-classes of the chromosomal parts.
Here, we simply worry about the instance-level values for particular genome
builds.
Given a chr band definition, the nested containment structures look like:
13q21.31 ==> 13q21.31, 13q21.3, 13q21, 13q2, 13q, 13
We determine the containing regions of the band by parsing the band-string;
since each alphanumeric is a significant "place", we can split it
with the shorter strings being parents of the longer string
Here we create build-specific chroms, which are instances of the classes
produced from ```Monochrom.py```.
You can instantiate any number of builds for a genome.
We leverage the Faldo model here for region definitions,
and map each of the chromosomal parts to SO.
We differentiate the build by adding the build id to the identifier prior
to the chromosome number.
These then are instances of the species-specific chromosomal class.
The build-specific chromosomes are created like:
<pre>
<build number>chr<num><band>
with triples for a given band like:
:hg19chr1p36.33 rdf[type] SO:chromosome_band, faldo:Region, CHR:9606chr1p36.33
:hg19chr1p36.33 subsequence_of :hg19chr1p36.3
:hg19chr1p36.33 faldo:location
[ a faldo:BothStrandPosition
faldo:begin 0,
faldo:end 2300000,
faldo:reference 'hg19']
</pre>
where any band in the file is an instance of a chr_band
(or a more specific type), is a subsequence of it's containing region, \
and is located in the specified coordinates.
We do not have a separate graph for testing.
TODO: any species by commandline argument
"""
files = {
# TODO accommodate multiple builds per species
'9606': {
'file': 'hg19cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/hg19/database/cytoBand.txt.gz',
'build_num': 'hg19',
'genome_label': 'Human'
},
'10090': {
'file': 'mm10cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/mm10/database/cytoBandIdeo.txt.gz',
'build_num': 'mm10',
'genome_label': 'Mouse'
},
# Note that there are no bands,
# arms or staining components for the species below at the moment
'7955': {
'file': 'danRer10cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/danRer10/database/cytoBandIdeo.txt.gz',
'build_num': 'danRer10',
'genome_label': 'Zebrafish'
},
'9913': {
'file': 'bosTau7cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/bosTau7/database/cytoBandIdeo.txt.gz',
'build_num': 'bosTau7',
'genome_label': 'cow'
},
'9031': {
'file': 'galGal4cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/galGal4/database/cytoBandIdeo.txt.gz',
'build_num': 'galGal4',
'genome_label': 'chicken'
},
'9823': {
'file': 'susScr3cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/susScr3/database/cytoBandIdeo.txt.gz',
'build_num': 'susScr3',
'genome_label': 'pig'
},
'9940': {
'file': 'oviAri3cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/oviAri3/database/cytoBandIdeo.txt.gz',
'build_num': 'oviAri3',
'genome_label': 'sheep'
},
'9796': {
'file': 'equCab2cytoBand.txt.gz',
'url': 'http://hgdownload.cse.ucsc.edu/goldenPath/equCab2/database/cytoBandIdeo.txt.gz',
'build_num': 'equCab2',
'genome_label': 'horse'
},
# TODO rainbow trout, 8022, when available
}
def __init__(self, tax_ids=None):
super().__init__('ucscbands')
self.tax_ids = tax_ids
self.load_bindings()
self.gu = GraphUtils(curie_map.get())
# Defaults
if self.tax_ids is None:
# self.tax_ids = [9606, 10090, 7955]
self.tax_ids = [9606, 10090, 7955, 9913, 9031, 9823, 9940, 9796]
# TODO add other species as defaults
self._check_tax_ids()
self.dataset = Dataset('ucscbands', 'UCSC Cytogenic Bands',
'http://hgdownload.cse.ucsc.edu', None,
'http://genome.ucsc.edu/license/')
# data-source specific warnings
# (will be removed when issues are cleared)
return
def fetch(self, is_dl_forced=False):
self.get_files(is_dl_forced)
return
def parse(self, limit=None):
if limit is not None:
logger.info("Only parsing first %d rows", limit)
logger.info("Parsing files...")
if self.testOnly:
self.testMode = True
for taxon in self.tax_ids:
self._get_chrbands(limit, str(taxon))
self._create_genome_builds()
self.load_core_bindings()
self.load_bindings()
# using the full graph as the test here
self.testgraph = self.graph
logger.info("Found %d nodes", len(self.graph))
logger.info("Done parsing files.")
return
def _get_chrbands(self, limit, taxon):
"""
:param limit:
:return:
"""
# TODO PYLINT figure out what limit was for and why it is unused
line_counter = 0
myfile = '/'.join((self.rawdir, self.files[taxon]['file']))
logger.info("Processing Chr bands from FILE: %s", myfile)
geno = Genotype(self.graph)
monochrom = Monochrom()
# used to hold band definitions for a chr
# in order to compute extent of encompasing bands
mybands = {}
# build the organism's genome from the taxon
genome_label = self.files[taxon]['genome_label']
taxon_id = 'NCBITaxon:'+taxon
# add the taxon as a class. adding the class label elsewhere
self.gu.addClassToGraph(self.graph, taxon_id, None)
self.gu.addSynonym(self.graph, taxon_id, genome_label)
self.gu.loadObjectProperties(self.graph, Feature.object_properties)
self.gu.loadProperties(self.graph, Feature.data_properties,
self.gu.DATAPROP)
self.gu.loadAllProperties(self.graph)
geno.addGenome(taxon_id, genome_label)
# add the build and the taxon it's in
build_num = self.files[taxon]['build_num']
build_id = 'UCSC:'+build_num
geno.addReferenceGenome(build_id, build_num, taxon_id)
# process the bands
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match('^#', line):
continue
# chr13 4500000 10000000 p12 stalk
(scaffold, start, stop, band_num, rtype) = line.split('\t')
line_counter += 1
# NOTE some less-finished genomes have
# placed and unplaced scaffolds
# * Placed scaffolds:
# the scaffolds have been placed within a chromosome.
# * Unlocalized scaffolds:
# although the chromosome within which the scaffold occurs
# is known, the scaffold's position or orientation
# is not known.
# * Unplaced scaffolds:
# it is not known which chromosome the scaffold belongs to
#
# find out if the thing is a full on chromosome, or a scaffold:
# ex: unlocalized scaffold: chr10_KL568008v1_random
# ex: unplaced scaffold: chrUn_AABR07022428v1
placed_scaffold_pattern = r'(chr(?:\d+|X|Y|Z|W|M))'
unlocalized_scaffold_pattern = placed_scaffold_pattern+r'_(\w+)_random'
unplaced_scaffold_pattern = r'chr(Un(?:_\w+)?)'
m = re.match(placed_scaffold_pattern+r'$', scaffold)
if m is not None and len(m.groups()) == 1:
# the chromosome is the first match of the pattern
chrom_num = m.group(1)
else:
# skip over anything that isn't a placed_scaffold
# at the class level
logger.info("Found non-placed chromosome %s", scaffold)
chrom_num = None
m_chr_unloc = re.match(unlocalized_scaffold_pattern, scaffold)
m_chr_unplaced = re.match(unplaced_scaffold_pattern, scaffold)
scaffold_num = None
if m:
pass
elif m_chr_unloc is not None and len(m_chr_unloc.groups()) == 2:
chrom_num = m_chr_unloc.group(1)
scaffold_num = chrom_num+'_'+m_chr_unloc.group(2)
elif m_chr_unplaced is not None and len(m_chr_unplaced.groups()) == 1:
scaffold_num = m_chr_unplaced.group(1)
else:
logger.error("There's a chr pattern that we aren't matching: %s",
scaffold)
if chrom_num is not None:
# the chrom class (generic) id
chrom_class_id = makeChromID(chrom_num, taxon, 'CHR')
# first, add the chromosome class (in the taxon)
geno.addChromosomeClass(chrom_num,
taxon_id, self.files[taxon]['genome_label'])
# then, add the chromosome instance (from the given build)
geno.addChromosomeInstance(chrom_num, build_id, build_num,
chrom_class_id)
# add the chr to the hashmap of coordinates for this build
# the chromosome coordinate space is itself
if chrom_num not in mybands.keys():
mybands[chrom_num] = {'min': 0,
'max': int(stop),
'chr': chrom_num,
'ref': build_id,
'parent': None,
'stain': None,
'type': Feature.types['chromosome']}
if scaffold_num is not None:
# this will put the coordinates of the scaffold
# in the scaffold-space and make sure that the scaffold
# is part of the correct parent.
# if chrom_num is None,
# then it will attach it to the genome,
# just like a reg chrom
mybands[scaffold_num] = {'min': start,
'max': stop,
'chr': scaffold_num,
'ref': build_id,
'parent': chrom_num,
'stain': None,
'type': Feature.types['assembly_component'],
'synonym': scaffold}
if band_num is not None and band_num.strip() != '':
# add the specific band
mybands[chrom_num+band_num] = {'min': start,
'max': stop,
'chr': chrom_num,
'ref': build_id,
'parent': None,
'stain': None,
'type': None}
# add the staining intensity of the band
if re.match(r'g(neg|pos|var)', rtype):
mybands[chrom_num+band_num]['stain'] = Feature.types.get(rtype)
# get the parent bands, and make them unique
parents = list(monochrom.make_parent_bands(band_num, set()))
# alphabetical sort will put them in smallest to biggest,
# so we reverse
parents.sort(reverse=True)
# print('parents of',chrom,band,':',parents)
if len(parents) > 0:
mybands[chrom_num+band_num]['parent'] = chrom_num+parents[0]
else:
# TODO PYLINT why is 'parent'
# a list() a couple of lines up and a set() here?
parents = set()
# loop through the parents and add them to the hash
# add the parents to the graph, in hierarchical order
# TODO PYLINT Consider using enumerate
# instead of iterating with range and len
for i in range(len(parents)):
rti = getChrPartTypeByNotation(parents[i])
pnum = chrom_num+parents[i]
sta = int(start)
sto = int(stop)
if pnum not in mybands.keys():
# add the parental band to the hash
b = {'min': min(sta, sto),
'max': max(sta, sto),
'chr': chrom_num,
'ref': build_id,
'parent': None,
'stain': None,
'type': rti}
mybands[pnum] = b
else:
# band already in the hash means it's a grouping band
# need to update the min/max coords
b = mybands.get(pnum)
b['min'] = min(sta, sto, b['min'])
b['max'] = max(sta, sto, b['max'])
mybands[pnum] = b
# also, set the max for the chrom
c = mybands.get(chrom_num)
c['max'] = max(sta, sto, c['max'])
mybands[chrom_num] = c
# add the parent relationships to each
if i < len(parents) - 1:
mybands[pnum]['parent'] = chrom_num+parents[i+1]
else:
# add the last one (p or q usually)
# as attached to the chromosome
mybands[pnum]['parent'] = chrom_num
f.close() # end looping through file
# loop through the hash and add the bands to the graph
for b in mybands.keys():
myband = mybands.get(b)
band_class_id = makeChromID(b, taxon, 'CHR')
band_class_label = makeChromLabel(b, genome_label)
band_build_id = makeChromID(b, build_num, 'MONARCH')
band_build_label = makeChromLabel(b, build_num)
# the build-specific chrom
chrom_in_build_id = makeChromID(myband['chr'], build_num, 'MONARCH')
# if it's != part, then add the class
if myband['type'] != Feature.types['assembly_component']:
self.gu.addClassToGraph(self.graph, band_class_id,
band_class_label, myband['type'])
bfeature = Feature(band_build_id, band_build_label,
band_class_id)
else:
bfeature = Feature(band_build_id, band_build_label,
myband['type'])
if 'synonym' in myband:
self.gu.addSynonym(self.graph, band_build_id,
myband['synonym'])
if myband['parent'] is None:
if myband['type'] == Feature.types['assembly_component']:
# since we likely don't know the chr,
# add it as a part of the build
geno.addParts(band_build_id, build_id)
elif myband['type'] == Feature.types['assembly_component']:
# geno.addParts(band_build_id, chrom_in_build_id)
parent_chrom_in_build = makeChromID(myband['parent'],
build_num, 'MONARCH')
bfeature.addSubsequenceOfFeature(self.graph,
parent_chrom_in_build)
# add the band as a feature
# (which also instantiates the owl:Individual)
bfeature.addFeatureStartLocation(myband['min'], chrom_in_build_id)
bfeature.addFeatureEndLocation(myband['max'], chrom_in_build_id)
if 'stain' in myband and myband['stain'] is not None:
# TODO TEC I recall 'has_staining_intensity' being dropped by MB
bfeature.addFeatureProperty(self.graph,
Feature.properties['has_staining_intensity'],
myband['stain'])
# type the band as a faldo:Region directly (add_region=False)
# bfeature.setNoBNodes(self.nobnodes)
# to come when we merge in ZFIN.py
bfeature.addFeatureToGraph(self.graph, False)
return
def _create_genome_builds(self):
"""
Various resources will map variations to either UCSC (hg*)
or to NCBI assemblies. Here we create the equivalences between them.
Data taken from:
https://genome.ucsc.edu/FAQ/FAQreleases.html#release1
:return:
"""
# TODO add more species
ucsc_assembly_id_map = {
"9606": {
"UCSC:hg38": "NCBIGenome:GRCh38",
"UCSC:hg19": "NCBIGenome:GRCh37",
"UCSC:hg18": "NCBIGenome:36.1",
"UCSC:hg17": "NCBIGenome:35",
"UCSC:hg16": "NCBIGenome:34",
"UCSC:hg15": "NCBIGenome:33",
},
"7955": {
"UCSC:danRer10": "NCBIGenome:GRCz10",
"UCSC:danRer7": "NCBIGenome:Zv9",
"UCSC:danRer6": "NCBIGenome:Zv8",
},
"10090": {
"UCSC:mm10": "NCBIGenome:GRCm38",
"UCSC:mm9": "NCBIGenome:37"
},
"9031": {
"UCSC:galGal4": "NCBIAssembly:317958",
},
"9913": {
"UCSC:bosTau7": "NCBIAssembly:GCF_000003205.5",
},
"9823": {
"UCSC:susScr3": "NCBIAssembly:304498",
},
"9940": {
"UCSC:oviAri3": "NCBIAssembly:GCF_000298735.1",
},
"9796": {
"UCSC:equCab2": "NCBIAssembly:GCF_000002305.2",
}
}
g = self.graph
geno = Genotype(g)
logger.info("Adding equivalent assembly identifiers")
for sp in ucsc_assembly_id_map:
tax_num = sp
tax_id = 'NCBITaxon:'+tax_num
mappings = ucsc_assembly_id_map[sp]
for i in mappings:
ucsc_id = i
ucsc_label = re.split(':', i)[1]
mapped_id = mappings[i]
mapped_label = re.split(':', mapped_id)[1]
mapped_label = 'NCBI build '+str(mapped_label)
geno.addReferenceGenome(ucsc_id, ucsc_label, tax_id)
geno.addReferenceGenome(mapped_id, mapped_label, tax_id)
self.gu.addSameIndividual(g, ucsc_id, mapped_id)
return
def _check_tax_ids(self):
for taxon in self.tax_ids:
if str(taxon) not in self.files:
raise Exception("Taxon " + str(taxon) + " not supported"
" by source UCSCBands")
def getTestSuite(self):
import unittest
from tests.test_ucscbands import UCSCBandsTestCase
test_suite = unittest.TestLoader().loadTestsFromTestCase(UCSCBandsTestCase)
return test_suite
def _get_variants(self, limit):
"""
Currently loops through the variant_summary file.
:param limit:
:return:
"""
gu = GraphUtils(curie_map.get())
if self.testMode:
g = self.testgraph
else:
g = self.graph
geno = Genotype(g)
gu.loadAllProperties(g)
f = Feature(None, None, None)
f.loadAllProperties(g)
gu.loadAllProperties(g)
# add the taxon and the genome
tax_num = '9606' # HARDCODE
tax_id = 'NCBITaxon:'+tax_num
tax_label = 'Human'
gu.addClassToGraph(g, tax_id, None)
geno.addGenome(tax_id, None) # label gets added elsewhere
# not unzipping the file
logger.info("Processing Variant records")
line_counter = 0
myfile = '/'.join((self.rawdir, self.files['variant_summary']['file']))
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match('^#', line):
continue
# AlleleID integer value as stored in the AlleleID field in ClinVar (//Measure/@ID in the XML)
# Type character, the type of variation
# Name character, the preferred name for the variation
# GeneID integer, GeneID in NCBI's Gene database
# GeneSymbol character, comma-separated list of GeneIDs overlapping the variation
# ClinicalSignificance character, comma-separated list of values of clinical significance reported for this variation
# for the mapping between the terms listed here and the integers in the .VCF files, see
# http://www.ncbi.nlm.nih.gov/clinvar/docs/clinsig/
# RS# (dbSNP) integer, rs# in dbSNP
# nsv (dbVar) character, the NSV identifier for the region in dbVar
# RCVaccession character, list of RCV accessions that report this variant
# TestedInGTR character, Y/N for Yes/No if there is a test registered as specific to this variation in the NIH Genetic Testing Registry (GTR)
# PhenotypeIDs character, list of db names and identifiers for phenotype(s) reported for this variant
# Origin character, list of all allelic origins for this variation
# Assembly character, name of the assembly on which locations are based
# Chromosome character, chromosomal location
# Start integer, starting location, in pter->qter orientation
# Stop integer, end location, in pter->qter orientation
# Cytogenetic character, ISCN band
# ReviewStatus character, highest review status for reporting this measure. For the key to the terms,
# and their relationship to the star graphics ClinVar displays on its web pages,
# see http://www.ncbi.nlm.nih.gov/clinvar/docs/variation_report/#interpretation
# HGVS(c.) character, RefSeq cDNA-based HGVS expression
# HGVS(p.) character, RefSeq protein-based HGVS expression
# NumberSubmitters integer, number of submissions with this variant
# LastEvaluated datetime, the latest time any submitter reported clinical significance
# Guidelines character, ACMG only right now, for the reporting of incidental variation in a Gene
# (NOTE: if ACMG, not a specific to the allele but to the Gene)
# OtherIDs character, list of other identifiers or sources of information about this variant
# VariantID integer, the value used to build the URL for the current default report,
# e.g. http://www.ncbi.nlm.nih.gov/clinvar/variation/1756/
#
# a crude check that there's an expected number of cols. if not, error out because something changed.
num_cols = len(line.split('\t'))
expected_numcols = 28
if num_cols != expected_numcols:
logger.error("Unexpected number of columns in raw file (%d actual vs %d expected)",
num_cols, expected_numcols)
(allele_num, allele_type, allele_name, gene_num, gene_symbol, clinical_significance,
dbsnp_num, dbvar_num, rcv_nums, tested_in_gtr, phenotype_ids, origin,
assembly, chr, start, stop, cytogenetic_loc,
review_status, hgvs_c, hgvs_p, number_of_submitters, last_eval,
guidelines, other_ids, variant_num, reference_allele, alternate_allele, categories) = line.split('\t')
# #### set filter=None in init if you don't want to have a filter
# if self.filter is not None:
# if ((self.filter == 'taxids' and (int(tax_num) not in self.tax_ids))
# or (self.filter == 'geneids' and (int(gene_num) not in self.gene_ids))):
# continue
# #### end filter
line_counter += 1
pheno_list = []
if phenotype_ids != '-':
# trim any leading/trailing semicolons/commas
phenotype_ids = re.sub('^[;,]', '', phenotype_ids)
phenotype_ids = re.sub('[;,]$', '', phenotype_ids)
pheno_list = re.split('[,;]', phenotype_ids)
if self.testMode:
# get intersection of test disease ids and these phenotype_ids
intersect = list(set([str(i) for i in self.disease_ids]) & set(pheno_list))
if int(gene_num) not in self.gene_ids and int(variant_num) not in self.variant_ids \
and len(intersect) < 1:
continue
# TODO may need to switch on assembly to create correct assembly/build identifiers
build_id = ':'.join(('NCBIGenome', assembly))
# make the reference genome build
geno.addReferenceGenome(build_id, assembly, tax_id)
allele_type_id = self._map_type_of_allele(allele_type)
bandinbuild_id = None
if str(chr) == '':
# check cytogenic location
if str(cytogenetic_loc).strip() != '':
# use cytogenic location to get the approximate location
# strangely, they still put an assembly number even when there's no numeric location
if not re.search('-',str(cytogenetic_loc)):
band_id = makeChromID(re.split('-',str(cytogenetic_loc)), tax_num, 'CHR')
geno.addChromosomeInstance(cytogenetic_loc, build_id, assembly, band_id)
bandinbuild_id = makeChromID(re.split('-',str(cytogenetic_loc)), assembly, 'MONARCH')
else:
# can't deal with ranges yet
pass
else:
# add the human chromosome class to the graph, and add the build-specific version of it
chr_id = makeChromID(str(chr), tax_num, 'CHR')
geno.addChromosomeClass(str(chr), tax_id, tax_label)
geno.addChromosomeInstance(str(chr), build_id, assembly, chr_id)
chrinbuild_id = makeChromID(str(chr), assembly, 'MONARCH')
seqalt_id = ':'.join(('ClinVarVariant', variant_num))
gene_id = None
if str(gene_num) != '-1' and str(gene_num) != 'more than 10': # they use -1 to indicate unknown gene
gene_id = ':'.join(('NCBIGene', str(gene_num)))
# FIXME there are some "variants" that are actually haplotypes
# probably will get taken care of when we switch to processing the xml
# for example, variant_num = 38562
# but there's no way to tell if it's a haplotype in the csv data
# so the dbsnp or dbvar should probably be primary, and the variant num be the vslc,
# with each of the dbsnps being added to it
# todo clinical significance needs to be mapped to a list of terms
# first, make the variant:
f = Feature(seqalt_id, allele_name, allele_type_id)
if start != '-' and start.strip() != '':
f.addFeatureStartLocation(start, chrinbuild_id)
if stop != '-' and stop.strip() != '':
f.addFeatureEndLocation(stop, chrinbuild_id)
f.addFeatureToGraph(g)
if bandinbuild_id is not None:
f.addSubsequenceOfFeature(g, bandinbuild_id)
# CHECK - this makes the assumption that there is only one affected chromosome per variant
# what happens with chromosomal rearrangement variants? shouldn't both chromosomes be here?
# add the hgvs as synonyms
if hgvs_c != '-' and hgvs_c.strip() != '':
gu.addSynonym(g, seqalt_id, hgvs_c)
if hgvs_p != '-' and hgvs_p.strip() != '':
gu.addSynonym(g, seqalt_id, hgvs_p)
# add the dbsnp and dbvar ids as equivalent
if dbsnp_num != '-' and int(dbsnp_num) != -1:
dbsnp_id = 'dbSNP:rs'+str(dbsnp_num)
gu.addIndividualToGraph(g, dbsnp_id, None)
gu.addSameIndividual(g, seqalt_id, dbsnp_id)
if dbvar_num != '-':
dbvar_id = 'dbVar:'+dbvar_num
gu.addIndividualToGraph(g, dbvar_id, None)
gu.addSameIndividual(g, seqalt_id, dbvar_id)
# TODO - not sure if this is right... add as xref?
# the rcv is like the combo of the phenotype with the variant
if rcv_nums != '-':
for rcv_num in re.split(';',rcv_nums):
rcv_id = 'ClinVar:'+rcv_num
gu.addIndividualToGraph(g, rcv_id, None)
gu.addXref(g, seqalt_id, rcv_id)
if gene_id is not None:
# add the gene
gu.addClassToGraph(g, gene_id, gene_symbol)
# make a variant locus
vl_id = '_'+gene_num+'-'+variant_num
if self.nobnodes:
vl_id = ':'+vl_id
vl_label = allele_name
gu.addIndividualToGraph(g, vl_id, vl_label, geno.genoparts['variant_locus'])
geno.addSequenceAlterationToVariantLocus(seqalt_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
# some basic reporting
gmatch = re.search('\(\w+\)', allele_name)
if gmatch is not None and len(gmatch.groups()) > 0:
logger.info("Gene found in allele label, but no id provided: %s", gmatch.group(1))
elif re.match('more than 10', gene_symbol):
logger.info("More than 10 genes found; need to process XML to fetch (variant=%d)", int(variant_num))
else:
logger.info("No gene listed for variant %d", int(variant_num))
# parse the list of "phenotypes" which are diseases. add them as an association
# ;GeneReviews:NBK1440,MedGen:C0392514,OMIM:235200,SNOMED CT:35400008;MedGen:C3280096,OMIM:614193;MedGen:CN034317,OMIM:612635;MedGen:CN169374
# the list is both semicolon delimited and comma delimited, but i don't know why!
# some are bad, like: Orphanet:ORPHA ORPHA319705,SNOMED CT:49049000
if phenotype_ids != '-':
for p in pheno_list:
m = re.match("(Orphanet:ORPHA(?:\s*ORPHA)?)", p)
if m is not None and len(m.groups()) > 0:
p = re.sub(m.group(1), 'Orphanet:', p.strip())
elif re.match('SNOMED CT', p):
p = re.sub('SNOMED CT', 'SNOMED', p.strip())
assoc = G2PAssoc(self.name, seqalt_id, p.strip())
assoc.add_association_to_graph(g)
if other_ids != '-':
id_list = other_ids.split(',')
# process the "other ids"
# ex: CFTR2:F508del,HGMD:CD890142,OMIM Allelic Variant:602421.0001
# TODO make more xrefs
for xrefid in id_list:
prefix = xrefid.split(':')[0].strip()
if prefix == 'OMIM Allelic Variant':
xrefid = 'OMIM:'+xrefid.split(':')[1]
gu.addIndividualToGraph(g, xrefid, None)
gu.addSameIndividual(g, seqalt_id, xrefid)
elif prefix == 'HGMD':
gu.addIndividualToGraph(g, xrefid, None)
gu.addSameIndividual(g, seqalt_id, xrefid)
elif prefix == 'dbVar' and dbvar_num == xrefid.split(':')[1].strip():
pass # skip over this one
elif re.search('\s', prefix):
pass
# logger.debug('xref prefix has a space: %s', xrefid)
else:
# should be a good clean prefix
# note that HGMD variants are in here as Xrefs because we can't resolve URIs for them
# logger.info("Adding xref: %s", xrefid)
# gu.addXref(g, seqalt_id, xrefid)
# logger.info("xref prefix to add: %s", xrefid)
pass
if not self.testMode and limit is not None and line_counter > limit:
break
gu.loadProperties(g, G2PAssoc.object_properties, gu.OBJPROP)
gu.loadProperties(g, G2PAssoc.annotation_properties, gu.ANNOTPROP)
gu.loadProperties(g, G2PAssoc.datatype_properties, gu.DATAPROP)
logger.info("Finished parsing variants")
return
