def _add_snp_to_graph(self,
snp_id,
snp_label,
chrom_num,
chrom_pos,
context,
risk_allele_frequency=None):
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
if chrom_num != '' and chrom_pos != '':
location = self._make_location_curie(chrom_num, chrom_pos)
if location not in self.id_location_map:
self.id_location_map[location] = set()
else:
location = None
alteration = re.search(r'-(.*)$', snp_id)
if alteration is not None and re.match(r'[ATGC]', alteration.group(1)):
# add variation to snp
pass # TODO
if location is not None:
self.id_location_map[location].add(snp_id)
# create the chromosome
chrom_id = makeChromID(chrom_num, self.localtt['reference assembly'],
'CHR')
# add the feature to the graph
snp_description = None
if risk_allele_frequency is not None\
and risk_allele_frequency != ''\
and risk_allele_frequency != 'NR':
snp_description = str(
risk_allele_frequency) + ' [risk allele frequency]'
feat = Feature(graph, snp_id, snp_label.strip(), self.globaltt['SNP'],
snp_description)
if chrom_num != '' and chrom_pos != '':
feat.addFeatureStartLocation(chrom_pos, chrom_id)
feat.addFeatureEndLocation(chrom_pos, chrom_id)
feat.addFeatureToGraph()
feat.addTaxonToFeature(self.globaltt['H**o sapiens'])
# TODO consider adding allele frequency as property;
# but would need background info to do that
# also want to add other descriptive info about
# the variant from the context
for ctx in re.split(r';', context):
ctx = ctx.strip()
cid = self.resolve(ctx, False)
if cid != ctx:
model.addType(snp_id, cid)
return
def _add_snp_to_graph(
self, snp_id, snp_label, chrom_num, chrom_pos, context,
risk_allele_frequency=None):
# constants
tax_id = 'NCBITaxon:9606'
genome_version = 'GRCh38'
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
if chrom_num != '' and chrom_pos != '':
location = self._make_location_curie(chrom_num, chrom_pos)
if location not in self.id_location_map:
self.id_location_map[location] = set()
else:
location = None
alteration = re.search(r'-(.*)$', snp_id)
if alteration is not None \
and re.match(r'[ATGC]', alteration.group(1)):
# add variation to snp
pass # TODO
if location is not None:
self.id_location_map[location].add(snp_id)
# create the chromosome
chrom_id = makeChromID(chrom_num, genome_version, 'CHR')
# add the feature to the graph
snp_description = None
if risk_allele_frequency is not None\
and risk_allele_frequency != ''\
and risk_allele_frequency != 'NR':
snp_description = \
str(risk_allele_frequency) + \
' [risk allele frequency]'
f = Feature(
g, snp_id, snp_label.strip(),
Feature.types['SNP'], snp_description)
if chrom_num != '' and chrom_pos != '':
f.addFeatureStartLocation(chrom_pos, chrom_id)
f.addFeatureEndLocation(chrom_pos, chrom_id)
f.addFeatureToGraph()
f.addTaxonToFeature(tax_id)
# TODO consider adding allele frequency as property;
# but would need background info to do that
# also want to add other descriptive info about
# the variant from the context
for c in re.split(r';', context):
cid = self._map_variant_type(c.strip())
if cid is not None:
model.addType(snp_id, cid)
return
def _add_snp_to_graph(
self, snp_id, snp_label, chrom_num, chrom_pos, context,
risk_allele_frequency=None):
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
if chrom_num != '' and chrom_pos != '':
location = self._make_location_curie(chrom_num, chrom_pos)
if location not in self.id_location_map:
self.id_location_map[location] = set()
else:
location = None
alteration = re.search(r'-(.*)$', snp_id)
if alteration is not None and re.match(r'[ATGC]', alteration.group(1)):
# add variation to snp
pass # TODO
if location is not None:
self.id_location_map[location].add(snp_id)
# create the chromosome
chrom_id = makeChromID(chrom_num, self.localtt['reference assembly'], 'CHR')
# add the feature to the graph
snp_description = None
if risk_allele_frequency is not None\
and risk_allele_frequency != ''\
and risk_allele_frequency != 'NR':
snp_description = str(risk_allele_frequency) + ' [risk allele frequency]'
feat = Feature(
graph, snp_id, snp_label.strip(), self.globaltt['SNP'], snp_description)
if chrom_num != '' and chrom_pos != '':
feat.addFeatureStartLocation(chrom_pos, chrom_id)
feat.addFeatureEndLocation(chrom_pos, chrom_id)
feat.addFeatureToGraph()
feat.addTaxonToFeature(self.globaltt['H**o sapiens'])
# TODO consider adding allele frequency as property;
# but would need background info to do that
# also want to add other descriptive info about
# the variant from the context
for ctx in re.split(r';', context):
ctx = ctx.strip()
cid = self.resolve(ctx, False)
if cid != ctx:
model.addType(snp_id, cid)
return
def _get_variants(self, limit):
"""
Currently loops through the variant_summary file.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
geno = Genotype(g)
f = Feature(g, None, None, None)
# add the taxon and the genome
tax_num = '9606' # HARDCODE
tax_id = 'NCBITaxon:'+tax_num
tax_label = 'Human'
model.addClassToGraph(tax_id, None)
geno.addGenome(tax_id, tax_label) # label gets added elsewhere
# not unzipping the file
logger.info("Processing Variant records")
line_counter = 0
myfile = '/'.join((self.rawdir, self.files['variant_summary']['file']))
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match(r'^#', line):
continue
# AlleleID integer value as stored in the AlleleID field in ClinVar (//Measure/@ID in the XML)
# Type character, the type of variation
# Name character, the preferred name for the variation
# GeneID integer, GeneID in NCBI's Gene database
# GeneSymbol character, comma-separated list of GeneIDs overlapping the variation
# ClinicalSignificance character, comma-separated list of values of clinical significance reported for this variation
# for the mapping between the terms listed here and the integers in the .VCF files, see
# http://www.ncbi.nlm.nih.gov/clinvar/docs/clinsig/
# RS# (dbSNP) integer, rs# in dbSNP
# nsv (dbVar) character, the NSV identifier for the region in dbVar
# RCVaccession character, list of RCV accessions that report this variant
# TestedInGTR character, Y/N for Yes/No if there is a test registered as specific to this variation in the NIH Genetic Testing Registry (GTR)
# PhenotypeIDs character, list of db names and identifiers for phenotype(s) reported for this variant
# Origin character, list of all allelic origins for this variation
# Assembly character, name of the assembly on which locations are based
# Chromosome character, chromosomal location
# Start integer, starting location, in pter->qter orientation
# Stop integer, end location, in pter->qter orientation
# Cytogenetic character, ISCN band
# ReviewStatus character, highest review status for reporting this measure. For the key to the terms,
# and their relationship to the star graphics ClinVar displays on its web pages,
# see http://www.ncbi.nlm.nih.gov/clinvar/docs/variation_report/#interpretation
# HGVS(c.) character, RefSeq cDNA-based HGVS expression
# HGVS(p.) character, RefSeq protein-based HGVS expression
# NumberSubmitters integer, number of submissions with this variant
# LastEvaluated datetime, the latest time any submitter reported clinical significance
# Guidelines character, ACMG only right now, for the reporting of incidental variation in a Gene
# (NOTE: if ACMG, not a specific to the allele but to the Gene)
# OtherIDs character, list of other identifiers or sources of information about this variant
# VariantID integer, the value used to build the URL for the current default report,
# e.g. http://www.ncbi.nlm.nih.gov/clinvar/variation/1756/
#
# a crude check that there's an expected number of cols.
# if not, error out because something changed.
num_cols = len(line.split('\t'))
expected_numcols = 29
if num_cols != expected_numcols:
logger.error(
"Unexpected number of columns in raw file " +
"(%d actual vs %d expected)",
num_cols, expected_numcols)
(allele_num, allele_type, allele_name, gene_num, gene_symbol,
clinical_significance, dbsnp_num, dbvar_num, rcv_nums,
tested_in_gtr, phenotype_ids, origin, assembly, chr, start,
stop, cytogenetic_loc, review_status, hgvs_c, hgvs_p,
number_of_submitters, last_eval, guidelines, other_ids,
variant_num, reference_allele, alternate_allele, categories,
ChromosomeAccession) = line.split('\t')
# ###set filter=None in init if you don't want to have a filter
# if self.filter is not None:
# if ((self.filter == 'taxids' and\
# (int(tax_num) not in self.tax_ids)) or\
# (self.filter == 'geneids' and\
# (int(gene_num) not in self.gene_ids))):
# continue
# #### end filter
line_counter += 1
pheno_list = []
if phenotype_ids != '-':
# trim any leading/trailing semicolons/commas
phenotype_ids = re.sub(r'^[;,]', '', phenotype_ids)
phenotype_ids = re.sub(r'[;,]$', '', phenotype_ids)
pheno_list = re.split(r'[,;]', phenotype_ids)
if self.testMode:
# get intersection of test disease ids
# and these phenotype_ids
intersect = \
list(
set([str(i)
for i in self.disease_ids]) & set(pheno_list))
if int(gene_num) not in self.gene_ids and\
int(variant_num) not in self.variant_ids and\
len(intersect) < 1:
continue
# TODO may need to switch on assembly to create correct
# assembly/build identifiers
build_id = ':'.join(('NCBIGenome', assembly))
# make the reference genome build
geno.addReferenceGenome(build_id, assembly, tax_id)
allele_type_id = self._map_type_of_allele(allele_type)
bandinbuild_id = None
if str(chr) == '':
# check cytogenic location
if str(cytogenetic_loc).strip() != '':
# use cytogenic location to get the apx location
# oddly, they still put an assembly number even when
# there's no numeric location
if not re.search(r'-', str(cytogenetic_loc)):
band_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)),
tax_num, 'CHR')
geno.addChromosomeInstance(
cytogenetic_loc, build_id, assembly, band_id)
bandinbuild_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)),
assembly, 'MONARCH')
else:
# can't deal with ranges yet
pass
else:
# add the human chromosome class to the graph,
# and add the build-specific version of it
chr_id = makeChromID(str(chr), tax_num, 'CHR')
geno.addChromosomeClass(str(chr), tax_id, tax_label)
geno.addChromosomeInstance(
str(chr), build_id, assembly, chr_id)
chrinbuild_id = makeChromID(str(chr), assembly, 'MONARCH')
seqalt_id = ':'.join(('ClinVarVariant', variant_num))
gene_id = None
# they use -1 to indicate unknown gene
if str(gene_num) != '-1' and str(gene_num) != 'more than 10':
if re.match(r'^Gene:', gene_num):
gene_num = "NCBI" + gene_num
else:
gene_id = ':'.join(('NCBIGene', str(gene_num)))
# FIXME there are some "variants" that are actually haplotypes
# probably will get taken care of when we switch to processing
# the xml for example, variant_num = 38562
# but there's no way to tell if it's a haplotype
# in the csv data so the dbsnp or dbvar
# should probably be primary,
# and the variant num be the vslc,
# with each of the dbsnps being added to it
# TODO clinical significance needs to be mapped to
# a list of terms
# first, make the variant:
f = Feature(seqalt_id, allele_name, allele_type_id)
if start != '-' and start.strip() != '':
f.addFeatureStartLocation(start, chrinbuild_id)
if stop != '-' and stop.strip() != '':
f.addFeatureEndLocation(stop, chrinbuild_id)
f.addFeatureToGraph()
f.addTaxonToFeature(tax_id)
# make the ClinVarVariant the clique leader
model.makeLeader(seqalt_id)
if bandinbuild_id is not None:
f.addSubsequenceOfFeature(bandinbuild_id)
# CHECK - this makes the assumption that there is
# only one affected chromosome per variant what happens with
# chromosomal rearrangement variants?
# shouldn't both chromosomes be here?
# add the hgvs as synonyms
if hgvs_c != '-' and hgvs_c.strip() != '':
model.addSynonym(seqalt_id, hgvs_c)
if hgvs_p != '-' and hgvs_p.strip() != '':
model.addSynonym(seqalt_id, hgvs_p)
# add the dbsnp and dbvar ids as equivalent
if dbsnp_num != '-' and int(dbsnp_num) != -1:
dbsnp_id = 'dbSNP:rs'+str(dbsnp_num)
model.addIndividualToGraph(dbsnp_id, None)
model.addSameIndividual(seqalt_id, dbsnp_id)
if dbvar_num != '-':
dbvar_id = 'dbVar:'+dbvar_num
model.addIndividualToGraph(dbvar_id, None)
model.addSameIndividual(seqalt_id, dbvar_id)
# TODO - not sure if this is right... add as xref?
# the rcv is like the combo of the phenotype with the variant
if rcv_nums != '-':
for rcv_num in re.split(r';', rcv_nums):
rcv_id = 'ClinVar:' + rcv_num
model.addIndividualToGraph(rcv_id, None)
model.addXref(seqalt_id, rcv_id)
if gene_id is not None:
# add the gene
model.addClassToGraph(gene_id, gene_symbol)
# make a variant locus
vl_id = '_'+gene_num+'-'+variant_num
if self.nobnodes:
vl_id = ':'+vl_id
vl_label = allele_name
model.addIndividualToGraph(
vl_id, vl_label, geno.genoparts['variant_locus'])
geno.addSequenceAlterationToVariantLocus(seqalt_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
# some basic reporting
gmatch = re.search(r'\(\w+\)', allele_name)
if gmatch is not None and len(gmatch.groups()) > 0:
logger.info(
"Gene found in allele label, but no id provided: %s",
gmatch.group(1))
elif re.match(r'more than 10', gene_symbol):
logger.info(
"More than 10 genes found; "
"need to process XML to fetch (variant=%d)",
int(variant_num))
else:
logger.info(
"No gene listed for variant %d",
int(variant_num))
# parse the list of "phenotypes" which are diseases.
# add them as an association
# ;GeneReviews:NBK1440,MedGen:C0392514,OMIM:235200,SNOMED CT:35400008;MedGen:C3280096,OMIM:614193;MedGen:CN034317,OMIM:612635;MedGen:CN169374
# the list is both semicolon delimited and comma delimited,
# but i don't know why! some are bad, like:
# Orphanet:ORPHA ORPHA319705,SNOMED CT:49049000
if phenotype_ids != '-':
for phenotype in pheno_list:
m = re.match(
r"(Orphanet:ORPHA(?:\s*ORPHA)?)", phenotype)
if m is not None and len(m.groups()) > 0:
phenotype = re.sub(
m.group(1), 'Orphanet:', phenotype.strip())
elif re.match(r'ORPHA:\d+', phenotype):
phenotype = re.sub(
r'^ORPHA', 'Orphanet', phenotype.strip())
elif re.match(r'Human Phenotype Ontology', phenotype):
phenotype = re.sub(
r'^Human Phenotype Ontology', '',
phenotype.strip())
elif re.match(r'SNOMED CT:\s?', phenotype):
phenotype = re.sub(
r'SNOMED CT:\s?', 'SNOMED:', phenotype.strip())
elif re.match(r'^Gene:', phenotype):
continue
assoc = G2PAssoc(
g, self.name, seqalt_id, phenotype.strip())
assoc.add_association_to_graph()
if other_ids != '-':
id_list = other_ids.split(',')
# process the "other ids" ex:
# CFTR2:F508del,HGMD:CD890142,OMIM Allelic Variant:602421.0001
# TODO make more xrefs
for xrefid in id_list:
prefix = xrefid.split(':')[0].strip()
if prefix == 'OMIM Allelic Variant':
xrefid = 'OMIM:'+xrefid.split(':')[1]
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'HGMD':
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'dbVar' \
and dbvar_num == xrefid.split(':')[1].strip():
pass # skip over this one
elif re.search(r'\s', prefix):
pass
# logger.debug(
# 'xref prefix has a space: %s', xrefid)
else:
# should be a good clean prefix
# note that HGMD variants are in here as Xrefs
# because we can't resolve URIs for them
# logger.info("Adding xref: %s", xrefid)
# gu.addXref(g, seqalt_id, xrefid)
# logger.info("xref prefix to add: %s", xrefid)
pass
if not self.testMode and limit is not None \
and line_counter > limit:
break
logger.info("Finished parsing variants")
return
def _get_variants(self, limit):
"""
Currently loops through the variant_summary file.
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
geno = Genotype(g)
f = Feature(g, None, None, None)
# add the taxon and the genome
tax_num = '9606' # HARDCODE
tax_id = 'NCBITaxon:' + tax_num
tax_label = 'Human'
model.addClassToGraph(tax_id, None)
geno.addGenome(tax_id, tax_label) # label gets added elsewhere
# not unzipping the file
logger.info("Processing Variant records")
line_counter = 0
myfile = '/'.join((self.rawdir, self.files['variant_summary']['file']))
with gzip.open(myfile, 'rb') as f:
for line in f:
# skip comments
line = line.decode().strip()
if re.match(r'^#', line):
continue
# AlleleID integer value as stored in the AlleleID field in ClinVar (//Measure/@ID in the XML)
# Type character, the type of variation
# Name character, the preferred name for the variation
# GeneID integer, GeneID in NCBI's Gene database
# GeneSymbol character, comma-separated list of GeneIDs overlapping the variation
# ClinicalSignificance character, comma-separated list of values of clinical significance reported for this variation
# for the mapping between the terms listed here and the integers in the .VCF files, see
# http://www.ncbi.nlm.nih.gov/clinvar/docs/clinsig/
# RS# (dbSNP) integer, rs# in dbSNP
# nsv (dbVar) character, the NSV identifier for the region in dbVar
# RCVaccession character, list of RCV accessions that report this variant
# TestedInGTR character, Y/N for Yes/No if there is a test registered as specific to this variation in the NIH Genetic Testing Registry (GTR)
# PhenotypeIDs character, list of db names and identifiers for phenotype(s) reported for this variant
# Origin character, list of all allelic origins for this variation
# Assembly character, name of the assembly on which locations are based
# Chromosome character, chromosomal location
# Start integer, starting location, in pter->qter orientation
# Stop integer, end location, in pter->qter orientation
# Cytogenetic character, ISCN band
# ReviewStatus character, highest review status for reporting this measure. For the key to the terms,
# and their relationship to the star graphics ClinVar displays on its web pages,
# see http://www.ncbi.nlm.nih.gov/clinvar/docs/variation_report/#interpretation
# HGVS(c.) character, RefSeq cDNA-based HGVS expression
# HGVS(p.) character, RefSeq protein-based HGVS expression
# NumberSubmitters integer, number of submissions with this variant
# LastEvaluated datetime, the latest time any submitter reported clinical significance
# Guidelines character, ACMG only right now, for the reporting of incidental variation in a Gene
# (NOTE: if ACMG, not a specific to the allele but to the Gene)
# OtherIDs character, list of other identifiers or sources of information about this variant
# VariantID integer, the value used to build the URL for the current default report,
# e.g. http://www.ncbi.nlm.nih.gov/clinvar/variation/1756/
#
# a crude check that there's an expected number of cols.
# if not, error out because something changed.
num_cols = len(line.split('\t'))
expected_numcols = 29
if num_cols != expected_numcols:
logger.error(
"Unexpected number of columns in raw file " +
"(%d actual vs %d expected)", num_cols,
expected_numcols)
(allele_num, allele_type, allele_name, gene_num, gene_symbol,
clinical_significance, dbsnp_num, dbvar_num, rcv_nums,
tested_in_gtr, phenotype_ids, origin, assembly, chr, start,
stop, cytogenetic_loc, review_status, hgvs_c, hgvs_p,
number_of_submitters, last_eval, guidelines, other_ids,
variant_num, reference_allele, alternate_allele, categories,
ChromosomeAccession) = line.split('\t')
# ###set filter=None in init if you don't want to have a filter
# if self.filter is not None:
# if ((self.filter == 'taxids' and\
# (int(tax_num) not in self.tax_ids)) or\
# (self.filter == 'geneids' and\
# (int(gene_num) not in self.gene_ids))):
# continue
# #### end filter
line_counter += 1
pheno_list = []
if phenotype_ids != '-':
# trim any leading/trailing semicolons/commas
phenotype_ids = re.sub(r'^[;,]', '', phenotype_ids)
phenotype_ids = re.sub(r'[;,]$', '', phenotype_ids)
pheno_list = re.split(r'[,;]', phenotype_ids)
if self.testMode:
# get intersection of test disease ids
# and these phenotype_ids
intersect = \
list(
set([str(i)
for i in self.disease_ids]) & set(pheno_list))
if int(gene_num) not in self.gene_ids and\
int(variant_num) not in self.variant_ids and\
len(intersect) < 1:
continue
# TODO may need to switch on assembly to create correct
# assembly/build identifiers
build_id = ':'.join(('NCBIGenome', assembly))
# make the reference genome build
geno.addReferenceGenome(build_id, assembly, tax_id)
allele_type_id = self._map_type_of_allele(allele_type)
bandinbuild_id = None
if str(chr) == '':
# check cytogenic location
if str(cytogenetic_loc).strip() != '':
# use cytogenic location to get the apx location
# oddly, they still put an assembly number even when
# there's no numeric location
if not re.search(r'-', str(cytogenetic_loc)):
band_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)), tax_num,
'CHR')
geno.addChromosomeInstance(cytogenetic_loc,
build_id, assembly,
band_id)
bandinbuild_id = makeChromID(
re.split(r'-', str(cytogenetic_loc)), assembly,
'MONARCH')
else:
# can't deal with ranges yet
pass
else:
# add the human chromosome class to the graph,
# and add the build-specific version of it
chr_id = makeChromID(str(chr), tax_num, 'CHR')
geno.addChromosomeClass(str(chr), tax_id, tax_label)
geno.addChromosomeInstance(str(chr), build_id, assembly,
chr_id)
chrinbuild_id = makeChromID(str(chr), assembly, 'MONARCH')
seqalt_id = ':'.join(('ClinVarVariant', variant_num))
gene_id = None
# they use -1 to indicate unknown gene
if str(gene_num) != '-1' and str(gene_num) != 'more than 10':
if re.match(r'^Gene:', gene_num):
gene_num = "NCBI" + gene_num
else:
gene_id = ':'.join(('NCBIGene', str(gene_num)))
# FIXME there are some "variants" that are actually haplotypes
# probably will get taken care of when we switch to processing
# the xml for example, variant_num = 38562
# but there's no way to tell if it's a haplotype
# in the csv data so the dbsnp or dbvar
# should probably be primary,
# and the variant num be the vslc,
# with each of the dbsnps being added to it
# TODO clinical significance needs to be mapped to
# a list of terms
# first, make the variant:
f = Feature(seqalt_id, allele_name, allele_type_id)
if start != '-' and start.strip() != '':
f.addFeatureStartLocation(start, chrinbuild_id)
if stop != '-' and stop.strip() != '':
f.addFeatureEndLocation(stop, chrinbuild_id)
f.addFeatureToGraph()
f.addTaxonToFeature(tax_id)
# make the ClinVarVariant the clique leader
model.makeLeader(seqalt_id)
if bandinbuild_id is not None:
f.addSubsequenceOfFeature(bandinbuild_id)
# CHECK - this makes the assumption that there is
# only one affected chromosome per variant what happens with
# chromosomal rearrangement variants?
# shouldn't both chromosomes be here?
# add the hgvs as synonyms
if hgvs_c != '-' and hgvs_c.strip() != '':
model.addSynonym(seqalt_id, hgvs_c)
if hgvs_p != '-' and hgvs_p.strip() != '':
model.addSynonym(seqalt_id, hgvs_p)
# add the dbsnp and dbvar ids as equivalent
if dbsnp_num != '-' and int(dbsnp_num) != -1:
dbsnp_id = 'dbSNP:rs' + str(dbsnp_num)
model.addIndividualToGraph(dbsnp_id, None)
model.addSameIndividual(seqalt_id, dbsnp_id)
if dbvar_num != '-':
dbvar_id = 'dbVar:' + dbvar_num
model.addIndividualToGraph(dbvar_id, None)
model.addSameIndividual(seqalt_id, dbvar_id)
# TODO - not sure if this is right... add as xref?
# the rcv is like the combo of the phenotype with the variant
if rcv_nums != '-':
for rcv_num in re.split(r';', rcv_nums):
rcv_id = 'ClinVar:' + rcv_num
model.addIndividualToGraph(rcv_id, None)
model.addXref(seqalt_id, rcv_id)
if gene_id is not None:
# add the gene
model.addClassToGraph(gene_id, gene_symbol)
# make a variant locus
vl_id = '_' + gene_num + '-' + variant_num
if self.nobnodes:
vl_id = ':' + vl_id
vl_label = allele_name
model.addIndividualToGraph(vl_id, vl_label,
geno.genoparts['variant_locus'])
geno.addSequenceAlterationToVariantLocus(seqalt_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
else:
# some basic reporting
gmatch = re.search(r'\(\w+\)', allele_name)
if gmatch is not None and len(gmatch.groups()) > 0:
logger.info(
"Gene found in allele label, but no id provided: %s",
gmatch.group(1))
elif re.match(r'more than 10', gene_symbol):
logger.info(
"More than 10 genes found; "
"need to process XML to fetch (variant=%d)",
int(variant_num))
else:
logger.info("No gene listed for variant %d",
int(variant_num))
# parse the list of "phenotypes" which are diseases.
# add them as an association
# ;GeneReviews:NBK1440,MedGen:C0392514,OMIM:235200,SNOMED CT:35400008;MedGen:C3280096,OMIM:614193;MedGen:CN034317,OMIM:612635;MedGen:CN169374
# the list is both semicolon delimited and comma delimited,
# but i don't know why! some are bad, like:
# Orphanet:ORPHA ORPHA319705,SNOMED CT:49049000
if phenotype_ids != '-':
for phenotype in pheno_list:
m = re.match(r"(Orphanet:ORPHA(?:\s*ORPHA)?)",
phenotype)
if m is not None and len(m.groups()) > 0:
phenotype = re.sub(m.group(1), 'Orphanet:',
phenotype.strip())
elif re.match(r'ORPHA:\d+', phenotype):
phenotype = re.sub(r'^ORPHA', 'Orphanet',
phenotype.strip())
elif re.match(r'Human Phenotype Ontology', phenotype):
phenotype = re.sub(r'^Human Phenotype Ontology',
'', phenotype.strip())
elif re.match(r'SNOMED CT:\s?', phenotype):
phenotype = re.sub(r'SNOMED CT:\s?', 'SNOMED:',
phenotype.strip())
elif re.match(r'^Gene:', phenotype):
continue
assoc = G2PAssoc(g, self.name, seqalt_id,
phenotype.strip())
assoc.add_association_to_graph()
if other_ids != '-':
id_list = other_ids.split(',')
# process the "other ids" ex:
# CFTR2:F508del,HGMD:CD890142,OMIM Allelic Variant:602421.0001
# TODO make more xrefs
for xrefid in id_list:
prefix = xrefid.split(':')[0].strip()
if prefix == 'OMIM Allelic Variant':
xrefid = 'OMIM:' + xrefid.split(':')[1]
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'HGMD':
model.addIndividualToGraph(xrefid, None)
model.addSameIndividual(seqalt_id, xrefid)
elif prefix == 'dbVar' \
and dbvar_num == xrefid.split(':')[1].strip():
pass # skip over this one
elif re.search(r'\s', prefix):
pass
# logger.debug(
# 'xref prefix has a space: %s', xrefid)
else:
# should be a good clean prefix
# note that HGMD variants are in here as Xrefs
# because we can't resolve URIs for them
# logger.info("Adding xref: %s", xrefid)
# gu.addXref(g, seqalt_id, xrefid)
# logger.info("xref prefix to add: %s", xrefid)
pass
if not self.testMode and limit is not None \
and line_counter > limit:
break
logger.info("Finished parsing variants")
return
def _process_qtls_genomic_location(
self, raw, txid, build_id, build_label, common_name, limit=None):
"""
This method
Triples created:
:param limit:
:return:
"""
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
line_counter = 0
geno = Genotype(graph)
# assume that chrs get added to the genome elsewhere
taxon_curie = 'NCBITaxon:' + txid
eco_id = self.globaltt['quantitative trait analysis evidence']
LOG.info("Processing QTL locations for %s from %s", taxon_curie, raw)
with gzip.open(raw, 'rt', encoding='ISO-8859-1') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for row in reader:
line_counter += 1
if re.match(r'^#', ' '.join(row)):
continue
(chromosome, qtl_source, qtl_type, start_bp, stop_bp, frame, strand,
score, attr) = row
example = '''
Chr.Z Animal QTLdb Production_QTL 33954873 34023581...
QTL_ID=2242;Name="Spleen percentage";Abbrev="SPLP";PUBMED_ID=17012160;trait_ID=2234;
trait="Spleen percentage";breed="leghorn";"FlankMarkers=ADL0022";VTO_name="spleen mass";
MO_name="spleen weight to body weight ratio";Map_Type="Linkage";Model="Mendelian";
Test_Base="Chromosome-wise";Significance="Significant";P-value="<0.05";F-Stat="5.52";
Variance="2.94";Dominance_Effect="-0.002";Additive_Effect="0.01
'''
str(example)
# make dictionary of attributes
# keys are:
# QTL_ID,Name,Abbrev,PUBMED_ID,trait_ID,trait,FlankMarkers,
# VTO_name,Map_Type,Significance,P-value,Model,
# Test_Base,Variance, Bayes-value,PTO_name,gene_IDsrc,peak_cM,
# CMO_name,gene_ID,F-Stat,LOD-score,Additive_Effect,
# Dominance_Effect,Likelihood_Ratio,LS-means,Breed,
# trait (duplicate with Name),Variance,Bayes-value,
# F-Stat,LOD-score,Additive_Effect,Dominance_Effect,
# Likelihood_Ratio,LS-means
# deal with poorly formed attributes
if re.search(r'"FlankMarkers";', attr):
attr = re.sub(r'FlankMarkers;', '', attr)
attr_items = re.sub(r'"', '', attr).split(";")
bad_attrs = set()
for attributes in attr_items:
if not re.search(r'=', attributes):
# remove this attribute from the list
bad_attrs.add(attributes)
attr_set = set(attr_items) - bad_attrs
attribute_dict = dict(item.split("=") for item in attr_set)
qtl_num = attribute_dict.get('QTL_ID')
if self.test_mode and int(qtl_num) not in self.test_ids:
continue
# make association between QTL and trait based on taxon
qtl_id = common_name + 'QTL:' + str(qtl_num)
model.addIndividualToGraph(qtl_id, None, self.globaltt['QTL'])
geno.addTaxon(taxon_curie, qtl_id)
#
trait_id = 'AQTLTrait:' + attribute_dict.get('trait_ID')
# if pub is in attributes, add it to the association
pub_id = None
if 'PUBMED_ID' in attribute_dict.keys():
pub_id = attribute_dict.get('PUBMED_ID')
if re.match(r'ISU.*', pub_id):
pub_id = 'AQTLPub:' + pub_id.strip()
reference = Reference(graph, pub_id)
else:
pub_id = 'PMID:' + pub_id.strip()
reference = Reference(
graph, pub_id, self.globaltt['journal article'])
reference.addRefToGraph()
# Add QTL to graph
assoc = G2PAssoc(
graph, self.name, qtl_id, trait_id,
self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
if 'P-value' in attribute_dict.keys():
scr = re.sub(r'<', '', attribute_dict.get('P-value'))
if ',' in scr:
scr = re.sub(r',', '.', scr)
if scr.isnumeric():
score = float(scr)
assoc.set_score(score)
assoc.add_association_to_graph()
# TODO make association to breed
# (which means making QTL feature in Breed background)
# get location of QTL
chromosome = re.sub(r'Chr\.', '', chromosome)
chrom_id = makeChromID(chromosome, taxon_curie, 'CHR')
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(
chromosome, build_id, build_label, chrom_id)
qtl_feature = Feature(graph, qtl_id, None, self.globaltt['QTL'])
if start_bp == '':
start_bp = None
qtl_feature.addFeatureStartLocation(
start_bp, chrom_in_build_id, strand,
[self.globaltt['FuzzyPosition']])
if stop_bp == '':
stop_bp = None
qtl_feature.addFeatureEndLocation(
stop_bp, chrom_in_build_id, strand,
[self.globaltt['FuzzyPosition']])
qtl_feature.addTaxonToFeature(taxon_curie)
qtl_feature.addFeatureToGraph()
if not self.test_mode and limit is not None and line_counter > limit:
break
# LOG.warning("Bad attribute flags in this file") # what does this even mean??
LOG.info("Done with QTL genomic mappings for %s", taxon_curie)
return
def _process_qtls_genetic_location(
self, raw, txid, common_name, limit=None):
"""
This function processes
Triples created:
:param limit:
:return:
"""
aql_curie = self.files[common_name + '_cm']['curie']
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
line_counter = 0
geno = Genotype(graph)
model = Model(graph)
eco_id = self.globaltt['quantitative trait analysis evidence']
taxon_curie = 'NCBITaxon:' + txid
LOG.info("Processing genetic location for %s from %s", taxon_curie, raw)
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(qtl_id,
qtl_symbol,
trait_name,
assotype,
empty,
chromosome,
position_cm,
range_cm,
flankmark_a2,
flankmark_a1,
peak_mark,
flankmark_b1,
flankmark_b2,
exp_id,
model_id,
test_base,
sig_level,
lod_score,
ls_mean,
p_values,
f_statistics,
variance,
bayes_value,
likelihood_ratio,
trait_id, dom_effect,
add_effect,
pubmed_id,
gene_id,
gene_id_src,
gene_id_type,
empty2) = row
if self.test_mode and int(qtl_id) not in self.test_ids:
continue
qtl_id = common_name + 'QTL:' + qtl_id.strip()
trait_id = ':'.join((aql_curie, trait_id.strip()))
# Add QTL to graph
feature = Feature(graph, qtl_id, qtl_symbol, self.globaltt['QTL'])
feature.addTaxonToFeature(taxon_curie)
# deal with the chromosome
chrom_id = makeChromID(chromosome, taxon_curie, 'CHR')
# add a version of the chromosome which is defined as
# the genetic map
build_id = 'MONARCH:'+common_name.strip()+'-linkage'
build_label = common_name+' genetic map'
geno.addReferenceGenome(build_id, build_label, taxon_curie)
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(
chromosome, build_id, build_label, chrom_id)
start = stop = None
# range_cm sometimes ends in "(Mb)" (i.e pig 2016 Nov)
range_mb = re.split(r'\(', range_cm)
if range_mb is not None:
range_cm = range_mb[0]
if re.search(r'[0-9].*-.*[0-9]', range_cm):
range_parts = re.split(r'-', range_cm)
# check for poorly formed ranges
if len(range_parts) == 2 and\
range_parts[0] != '' and range_parts[1] != '':
(start, stop) = [
int(float(x.strip())) for x in re.split(r'-', range_cm)]
else:
LOG.info(
"A cM range we can't handle for QTL %s: %s",
qtl_id, range_cm)
elif position_cm != '':
match = re.match(r'([0-9]*\.[0-9]*)', position_cm)
if match is not None:
position_cm = match.group()
start = stop = int(float(position_cm))
# FIXME remove converion to int for start/stop
# when schema can handle floats add in the genetic location
# based on the range
feature.addFeatureStartLocation(
start, chrom_in_build_id, None,
[self.globaltt['FuzzyPosition']])
feature.addFeatureEndLocation(
stop, chrom_in_build_id, None,
[self.globaltt['FuzzyPosition']])
feature.addFeatureToGraph()
# sometimes there's a peak marker, like a rsid.
# we want to add that as a variant of the gene,
# and xref it to the qtl.
dbsnp_id = None
if peak_mark != '' and peak_mark != '.' and \
re.match(r'rs', peak_mark.strip()):
dbsnp_id = 'dbSNP:'+peak_mark.strip()
model.addIndividualToGraph(
dbsnp_id, None,
self.globaltt['sequence_alteration'])
model.addXref(qtl_id, dbsnp_id)
gene_id = gene_id.replace('uncharacterized ', '').strip()
if gene_id is not None and gene_id != '' and gene_id != '.'\
and re.fullmatch(r'[^ ]*', gene_id) is not None:
# we assume if no src is provided and gene_id is an integer,
# then it is an NCBI gene ... (okay, lets crank that back a notch)
if gene_id_src == '' and gene_id.isdigit() and \
gene_id in self.gene_info:
# LOG.info(
# 'Warm & Fuzzy saying %s is a NCBI gene for %s',
# gene_id, common_name)
gene_id_src = 'NCBIgene'
elif gene_id_src == '' and gene_id.isdigit():
LOG.warning(
'Cold & Prickely saying %s is a NCBI gene for %s',
gene_id, common_name)
gene_id_src = 'NCBIgene'
elif gene_id_src == '':
LOG.error(
' "%s" is a NOT NCBI gene for %s', gene_id, common_name)
gene_id_src = None
if gene_id_src == 'NCBIgene':
gene_id = 'NCBIGene:' + gene_id
# we will expect that these will get labels elsewhere
geno.addGene(gene_id, None)
# FIXME what is the right relationship here?
geno.addAffectedLocus(qtl_id, gene_id)
if dbsnp_id is not None:
# add the rsid as a seq alt of the gene_id
vl_id = '_:' + re.sub(
r':', '', gene_id) + '-' + peak_mark.strip()
geno.addSequenceAlterationToVariantLocus(
dbsnp_id, vl_id)
geno.addAffectedLocus(vl_id, gene_id)
# add the trait
model.addClassToGraph(trait_id, trait_name)
# Add publication
reference = None
if re.match(r'ISU.*', pubmed_id):
pub_id = 'AQTLPub:'+pubmed_id.strip()
reference = Reference(graph, pub_id)
elif pubmed_id != '':
pub_id = 'PMID:' + pubmed_id.strip()
reference = Reference(
graph, pub_id, self.globaltt['journal article'])
if reference is not None:
reference.addRefToGraph()
# make the association to the QTL
assoc = G2PAssoc(
graph, self.name, qtl_id, trait_id, self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id as evidence
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
scr = re.sub(r'<', '', p_values)
scr = re.sub(r',', '.', scr) # international notation
if scr.isnumeric():
score = float(scr)
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph()
# make the association to the dbsnp_id, if found
if dbsnp_id is not None:
# make the association to the dbsnp_id
assoc = G2PAssoc(
graph, self.name, dbsnp_id, trait_id,
self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
scr = re.sub(r'<', '', p_values)
scr = re.sub(r',', '.', scr)
if scr.isnumeric():
score = float(scr)
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph()
if not self.test_mode and limit is not None and line_counter > limit:
break
LOG.info("Done with QTL genetic info")
return
def _process_qtls_genomic_location(
self, raw, src_key, txid, build_id, build_label, common_name, limit=None):
"""
This method
Triples created:
:param limit:
:return:
"""
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
model = Model(graph)
geno = Genotype(graph)
# assume that chrs get added to the genome elsewhere
taxon_curie = 'NCBITaxon:' + txid
eco_id = self.globaltt['quantitative trait analysis evidence']
LOG.info("Processing QTL locations for %s from %s", taxon_curie, raw)
with gzip.open(raw, 'rt', encoding='ISO-8859-1') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
# no header in GFF, so no header checking
col = self.files[src_key]['columns']
col_len = len(col)
for row in reader:
if row[0][0] == '#':
# LOG.info(row)
continue
if len(row) != col_len and ''.join(row[col_len:]) != '':
LOG.warning(
"Problem parsing in %s row %s\n"
"got %s cols but expected %s",
raw, reader.line_num, len(row), col_len)
LOG.info(row)
continue
chromosome = row[col.index('SEQNAME')].strip()
# qtl_source = row[col.index('SOURCE')].strip()
# qtl_type = row[col.index('FEATURE')].strip()
start_bp = row[col.index('START')].strip()
stop_bp = row[col.index('END')].strip()
# score = row[col.index('SCORE')].strip()
strand = row[col.index('STRAND')].strip()
# frame = row[col.index('FRAME')].strip()
attr = row[col.index('ATTRIBUTE')].strip()
example = '''
Chr.Z Animal QTLdb Production_QTL 33954873 34023581...
QTL_ID=2242;Name="Spleen percentage";Abbrev="SPLP";PUBMED_ID=17012160;trait_ID=2234;
trait="Spleen percentage";breed="leghorn";"FlankMarkers=ADL0022";VTO_name="spleen mass";
MO_name="spleen weight to body weight ratio";Map_Type="Linkage";Model="Mendelian";
Test_Base="Chromosome-wise";Significance="Significant";P-value="<0.05";F-Stat="5.52";
Variance="2.94";Dominance_Effect="-0.002";Additive_Effect="0.01
'''
str(example)
# make dictionary of attributes
# keys are:
# QTL_ID,Name,Abbrev,PUBMED_ID,trait_ID,trait,FlankMarkers,
# VTO_name,Map_Type,Significance,P-value,Model,
# Test_Base,Variance, Bayes-value,PTO_name,gene_IDsrc,peak_cM,
# CMO_name,gene_ID,F-Stat,LOD-score,Additive_Effect,
# Dominance_Effect,Likelihood_Ratio,LS-means,Breed,
# trait (duplicate with Name),Variance,Bayes-value,
# F-Stat,LOD-score,Additive_Effect,Dominance_Effect,
# Likelihood_Ratio,LS-means
# deal with poorly formed attributes
if re.search(r'"FlankMarkers";', attr):
attr = re.sub(r'FlankMarkers;', '', attr)
attr_items = re.sub(r'"', '', attr).split(";")
bad_attrs = set()
for attributes in attr_items:
if not re.search(r'=', attributes):
# remove this attribute from the list
bad_attrs.add(attributes)
attr_set = set(attr_items) - bad_attrs
attribute_dict = dict(item.split("=") for item in attr_set)
qtl_num = attribute_dict.get('QTL_ID')
if self.test_mode and int(qtl_num) not in self.test_ids:
continue
# make association between QTL and trait based on taxon
qtl_id = common_name + 'QTL:' + str(qtl_num)
model.addIndividualToGraph(qtl_id, None, self.globaltt['QTL'])
geno.addTaxon(taxon_curie, qtl_id)
#
trait_id = 'AQTLTrait:' + attribute_dict.get('trait_ID')
# if pub is in attributes, add it to the association
pub_id = None
if 'PUBMED_ID' in attribute_dict.keys():
pub_id = attribute_dict.get('PUBMED_ID')
if re.match(r'ISU.*', pub_id):
pub_id = 'AQTLPub:' + pub_id.strip()
reference = Reference(graph, pub_id)
else:
pub_id = 'PMID:' + pub_id.strip()
reference = Reference(
graph, pub_id, self.globaltt['journal article'])
reference.addRefToGraph()
# Add QTL to graph
assoc = G2PAssoc(
graph, self.name, qtl_id, trait_id,
self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
if 'P-value' in attribute_dict.keys():
scr = re.sub(r'<', '', attribute_dict.get('P-value'))
if ',' in scr:
scr = re.sub(r',', '.', scr)
if scr.isnumeric():
score = float(scr)
assoc.set_score(score)
assoc.add_association_to_graph()
# TODO make association to breed
# (which means making QTL feature in Breed background)
# get location of QTL
chromosome = re.sub(r'Chr\.', '', chromosome)
chrom_id = makeChromID(chromosome, taxon_curie, 'CHR')
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(
chromosome, build_id, build_label, chrom_id)
qtl_feature = Feature(graph, qtl_id, None, self.globaltt['QTL'])
if start_bp == '':
start_bp = None
qtl_feature.addFeatureStartLocation(
start_bp, chrom_in_build_id, strand,
[self.globaltt['FuzzyPosition']])
if stop_bp == '':
stop_bp = None
qtl_feature.addFeatureEndLocation(
stop_bp, chrom_in_build_id, strand,
[self.globaltt['FuzzyPosition']])
qtl_feature.addTaxonToFeature(taxon_curie)
qtl_feature.addFeatureToGraph()
if not self.test_mode and limit is not None and reader.line_num > limit:
break
# LOG.warning("Bad attribute flags in this file") # what does this even mean??
LOG.info("Done with QTL genomic mappings for %s", taxon_curie)
def _process_qtls_genetic_location(
self, raw, src_key, txid, common_name, limit=None):
"""
This function processes
Triples created:
:param limit:
:return:
"""
aql_curie = self.files[src_key]['curie']
common_name = common_name.strip()
if self.test_mode:
graph = self.testgraph
else:
graph = self.graph
geno = Genotype(graph)
model = Model(graph)
eco_id = self.globaltt['quantitative trait analysis evidence']
taxon_curie = 'NCBITaxon:' + txid
LOG.info("Processing genetic location for %s from %s", taxon_curie, raw)
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
reader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
# no header in these files, so no header checking
col = self.files[src_key]['columns']
col_len = len(col)
for row in reader:
if len(row) != col_len and ''.join(row[col_len:]) != '':
LOG.warning(
"Problem parsing %s line %i containing: \n%s\n"
"got %i cols but expected %i",
raw, reader.line_num, row, len(row), col_len)
# LOG.info(row)
continue
qtl_id = row[col.index('QTL_ID')].strip()
qtl_symbol = row[col.index('QTL_symbol')].strip()
trait_name = row[col.index('Trait_name')].strip()
# assotype = row[col.index('assotype')].strip()
chromosome = row[col.index('Chromosome')].strip()
position_cm = row[col.index('Position_cm')].strip()
range_cm = row[col.index('range_cm')].strip()
# flankmark_a2 = row[col.index('FlankMark_A2')].strip()
# flankmark_a1 = row[col.index('FlankMark_A1')].strip()
peak_mark = row[col.index('Peak_Mark')].strip()
# flankmark_b1 = row[col.index('FlankMark_B1')].strip()
# flankmark_b2 = row[col.index('FlankMark_B2')].strip()
# exp_id = row[col.index('Exp_ID')].strip()
# model_id = row[col.index('Model')].strip()
# test_base = row[col.index('testbase')].strip()
# sig_level = row[col.index('siglevel')].strip()
# lod_score = row[col.index('LOD_score')].strip()
# ls_mean = row[col.index('LS_mean')].strip()
p_values = row[col.index('P_values')].strip()
# f_statistics = row[col.index('F_Statistics')].strip()
# variance = row[col.index('VARIANCE')].strip()
# bayes_value = row[col.index('Bayes_value')].strip()
# likelihood_ratio = row[col.index('LikelihoodR')].strip()
trait_id = row[col.index('TRAIT_ID')].strip()
# dom_effect = row[col.index('Dom_effect')].strip()
# add_effect = row[col.index('Add_effect')].strip()
pubmed_id = row[col.index('PUBMED_ID')].strip()
gene_id = row[col.index('geneID')].strip()
gene_id_src = row[col.index('geneIDsrc')].strip()
# gene_id_type = row[col.index('geneIDtype')].strip()
if self.test_mode and int(qtl_id) not in self.test_ids:
continue
qtl_id = common_name + 'QTL:' + qtl_id.strip()
trait_id = ':'.join((aql_curie, trait_id.strip()))
# Add QTL to graph
feature = Feature(graph, qtl_id, qtl_symbol, self.globaltt['QTL'])
feature.addTaxonToFeature(taxon_curie)
# deal with the chromosome
chrom_id = makeChromID(chromosome, taxon_curie, 'CHR')
# add a version of the chromosome which is defined as
# the genetic map
build_id = 'MONARCH:' + common_name + '-linkage'
build_label = common_name + ' genetic map'
geno.addReferenceGenome(build_id, build_label, taxon_curie)
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(
chromosome, build_id, build_label, chrom_id)
start = stop = None
# range_cm sometimes ends in "(Mb)" (i.e pig 2016 Nov)
range_mb = re.split(r'\(', range_cm)
if range_mb is not None:
range_cm = range_mb[0]
if re.search(r'[0-9].*-.*[0-9]', range_cm):
range_parts = re.split(r'-', range_cm)
# check for poorly formed ranges
if len(range_parts) == 2 and\
range_parts[0] != '' and range_parts[1] != '':
(start, stop) = [
int(float(x.strip())) for x in re.split(r'-', range_cm)]
else:
LOG.info(
"A cM range we can't handle for QTL %s: %s",
qtl_id, range_cm)
elif position_cm != '':
match = re.match(r'([0-9]*\.[0-9]*)', position_cm)
if match is not None:
position_cm = match.group()
start = stop = int(float(position_cm))
# FIXME remove converion to int for start/stop
# when schema can handle floats add in the genetic location
# based on the range
feature.addFeatureStartLocation(
start, chrom_in_build_id, None,
[self.globaltt['FuzzyPosition']])
feature.addFeatureEndLocation(
stop, chrom_in_build_id, None,
[self.globaltt['FuzzyPosition']])
feature.addFeatureToGraph()
# sometimes there's a peak marker, like a rsid.
# we want to add that as a variant of the gene,
# and xref it to the qtl.
dbsnp_id = None
if peak_mark != '' and peak_mark != '.' and \
re.match(r'rs', peak_mark.strip()):
dbsnp_id = 'dbSNP:' + peak_mark.strip()
model.addIndividualToGraph(
dbsnp_id, None, self.globaltt['sequence_alteration'])
model.addXref(
qtl_id, dbsnp_id, xref_category=blv.terms['SequenceVariant'])
gene_id = gene_id.replace('uncharacterized ', '').strip()
gene_id = gene_id.strip(',') # for "100157483," in pig_QTLdata.txt
if gene_id is not None and gene_id != '' and gene_id != '.'\
and re.fullmatch(r'[^ ]*', gene_id) is not None:
# we assume if no src is provided and gene_id is an integer,
# then it is an NCBI gene ... (okay, lets crank that back a notch)
if gene_id_src == '' and gene_id.isdigit() and \
gene_id in self.gene_info:
# LOG.info(
# 'Warm & Fuzzy saying %s is a NCBI gene for %s',
# gene_id, common_name)
gene_id_src = 'NCBIgene'
elif gene_id_src == '' and gene_id.isdigit():
LOG.warning(
'Cold & Prickely saying %s is a NCBI gene for %s',
gene_id, common_name)
gene_id_src = 'NCBIgene'
elif gene_id_src == '':
LOG.error(
' "%s" is a NOT NCBI gene for %s', gene_id, common_name)
gene_id_src = None
if gene_id_src == 'NCBIgene':
gene_id = 'NCBIGene:' + gene_id
# we will expect that these will get labels elsewhere
geno.addGene(gene_id, None)
# FIXME what is the right relationship here?
geno.addAffectedLocus(qtl_id, gene_id)
if dbsnp_id is not None:
# add the rsid as a seq alt of the gene_id as a bnode
vl_id = self.make_id(re.sub(
r':', '', gene_id) + '-' + peak_mark.strip(), '_')
geno.addSequenceAlterationToVariantLocus(dbsnp_id, vl_id)
geno.addAffectedLocus(vl_id, gene_id)
# add the trait
model.addClassToGraph(
trait_id,
trait_name,
class_category=blv.terms['PhenotypicFeature'])
# Add publication
reference = None
if re.match(r'ISU.*', pubmed_id):
pub_id = 'AQTLPub:' + pubmed_id.strip()
reference = Reference(graph, pub_id)
elif pubmed_id != '':
pub_id = 'PMID:' + pubmed_id.strip()
reference = Reference(
graph, pub_id, self.globaltt['journal article'])
if reference is not None:
reference.addRefToGraph()
# make the association to the QTL
assoc = G2PAssoc(
graph, self.name, qtl_id, trait_id, self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id as evidence
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
scr = re.sub(r'<', '', p_values)
scr = re.sub(r',', '.', scr) # international notation
if scr.isnumeric():
score = float(scr)
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph()
# make the association to the dbsnp_id, if found
if dbsnp_id is not None:
# make the association to the dbsnp_id
assoc = G2PAssoc(
graph, self.name, dbsnp_id, trait_id,
self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
scr = re.sub(r'<', '', p_values)
scr = re.sub(r',', '.', scr)
if scr.isnumeric():
score = float(scr)
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph()
# off by one - the following actually gives us (limit + 1) records
if not self.test_mode and limit is not None and reader.line_num > limit:
break
LOG.info("Done with QTL genetic info")
def _process_qtls_genetic_location(
self, raw, txid, common_name, limit=None):
"""
This function processes
Triples created:
:param limit:
:return:
"""
if self.testMode:
graph = self.testgraph
else:
graph = self.graph
line_counter = 0
geno = Genotype(graph)
model = Model(graph)
eco_id = self.globaltt['quantitative trait analysis evidence']
taxon_curie = 'NCBITaxon:' + txid
LOG.info("Processing genetic location for %s from %s", taxon_curie, raw)
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(qtl_id,
qtl_symbol,
trait_name,
assotype,
empty,
chromosome,
position_cm,
range_cm,
flankmark_a2,
flankmark_a1,
peak_mark,
flankmark_b1,
flankmark_b2,
exp_id,
model_id,
test_base,
sig_level,
lod_score,
ls_mean,
p_values,
f_statistics,
variance,
bayes_value,
likelihood_ratio,
trait_id, dom_effect,
add_effect,
pubmed_id,
gene_id,
gene_id_src,
gene_id_type,
empty2) = row
if self.testMode and int(qtl_id) not in self.test_ids:
continue
qtl_id = common_name + 'QTL:' + qtl_id.strip()
trait_id = 'AQTLTrait:' + trait_id.strip()
# Add QTL to graph
feature = Feature(graph, qtl_id, qtl_symbol, self.globaltt['QTL'])
feature.addTaxonToFeature(taxon_curie)
# deal with the chromosome
chrom_id = makeChromID(chromosome, taxon_curie, 'CHR')
# add a version of the chromosome which is defined as
# the genetic map
build_id = 'MONARCH:'+common_name.strip()+'-linkage'
build_label = common_name+' genetic map'
geno.addReferenceGenome(build_id, build_label, taxon_curie)
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(
chromosome, build_id, build_label, chrom_id)
start = stop = None
# range_cm sometimes ends in "(Mb)" (i.e pig 2016 Nov)
range_mb = re.split(r'\(', range_cm)
if range_mb is not None:
range_cm = range_mb[0]
if re.search(r'[0-9].*-.*[0-9]', range_cm):
range_parts = re.split(r'-', range_cm)
# check for poorly formed ranges
if len(range_parts) == 2 and\
range_parts[0] != '' and range_parts[1] != '':
(start, stop) = [
int(float(x.strip())) for x in re.split(r'-', range_cm)]
else:
LOG.info(
"A cM range we can't handle for QTL %s: %s",
qtl_id, range_cm)
elif position_cm != '':
match = re.match(r'([0-9]*\.[0-9]*)', position_cm)
if match is not None:
position_cm = match.group()
start = stop = int(float(position_cm))
# FIXME remove converion to int for start/stop
# when schema can handle floats add in the genetic location
# based on the range
feature.addFeatureStartLocation(
start, chrom_in_build_id, None,
[self.globaltt['FuzzyPosition']])
feature.addFeatureEndLocation(
stop, chrom_in_build_id, None,
[self.globaltt['FuzzyPosition']])
feature.addFeatureToGraph()
# sometimes there's a peak marker, like a rsid.
# we want to add that as a variant of the gene,
# and xref it to the qtl.
dbsnp_id = None
if peak_mark != '' and peak_mark != '.' and \
re.match(r'rs', peak_mark.strip()):
dbsnp_id = 'dbSNP:'+peak_mark.strip()
model.addIndividualToGraph(
dbsnp_id, None,
self.globaltt['sequence_alteration'])
model.addXref(qtl_id, dbsnp_id)
gene_id = gene_id.replace('uncharacterized ', '').strip()
if gene_id is not None and gene_id != '' and gene_id != '.'\
and re.fullmatch(r'[^ ]*', gene_id) is not None:
# we assume if no src is provided and gene_id is an integer,
# then it is an NCBI gene ... (okay, lets crank that back a notch)
if gene_id_src == '' and gene_id.isdigit() and \
gene_id in self.gene_info:
# LOG.info(
# 'Warm & Fuzzy saying %s is a NCBI gene for %s',
# gene_id, common_name)
gene_id_src = 'NCBIgene'
elif gene_id_src == '' and gene_id.isdigit():
LOG.warning(
'Cold & Prickely saying %s is a NCBI gene for %s',
gene_id, common_name)
gene_id_src = 'NCBIgene'
elif gene_id_src == '':
LOG.error(
' "%s" is a NOT NCBI gene for %s', gene_id, common_name)
gene_id_src = None
if gene_id_src == 'NCBIgene':
gene_id = 'NCBIGene:' + gene_id
# we will expect that these will get labels elsewhere
geno.addGene(gene_id, None)
# FIXME what is the right relationship here?
geno.addAffectedLocus(qtl_id, gene_id)
if dbsnp_id is not None:
# add the rsid as a seq alt of the gene_id
vl_id = '_:' + re.sub(
r':', '', gene_id) + '-' + peak_mark.strip()
geno.addSequenceAlterationToVariantLocus(
dbsnp_id, vl_id)
geno.addAffectedLocus(vl_id, gene_id)
# add the trait
model.addClassToGraph(trait_id, trait_name)
# Add publication
reference = None
if re.match(r'ISU.*', pubmed_id):
pub_id = 'AQTLPub:'+pubmed_id.strip()
reference = Reference(graph, pub_id)
elif pubmed_id != '':
pub_id = 'PMID:' + pubmed_id.strip()
reference = Reference(
graph, pub_id, self.globaltt['journal article'])
if reference is not None:
reference.addRefToGraph()
# make the association to the QTL
assoc = G2PAssoc(
graph, self.name, qtl_id, trait_id, self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id as evidence
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
scr = re.sub(r'<', '', p_values)
scr = re.sub(r',', '.', scr) # international notation
if scr.isnumeric():
score = float(scr)
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph()
# make the association to the dbsnp_id, if found
if dbsnp_id is not None:
# make the association to the dbsnp_id
assoc = G2PAssoc(
graph, self.name, dbsnp_id, trait_id,
self.globaltt['is marker for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
scr = re.sub(r'<', '', p_values)
scr = re.sub(r',', '.', scr)
if scr.isnumeric():
score = float(scr)
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph()
if not self.testMode and limit is not None and line_counter > limit:
break
LOG.info("Done with QTL genetic info")
return
def _process_QTLs_genomic_location(
self, raw, taxon_id, build_id, build_label, limit=None):
"""
This method
Triples created:
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
model = Model(g)
line_counter = 0
geno = Genotype(g)
# assume that chrs get added to the genome elsewhere
# genome_id = geno.makeGenomeID(taxon_id) # TODO unused
eco_id = "ECO:0000061" # Quantitative Trait Analysis Evidence
logger.info("Processing QTL locations for %s", taxon_id)
with gzip.open(raw, 'rt', encoding='ISO-8859-1') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
# bad_attr_flag = False # TODO unused
for row in reader:
line_counter += 1
if re.match(r'^#', ' '.join(row)):
continue
(chromosome, qtl_source, qtl_type, start_bp, stop_bp, frame,
strand, score, attr) = row
# Chr.Z Animal QTLdb Production_QTL 33954873 34023581 . . .
# QTL_ID=2242;Name="Spleen percentage";Abbrev="SPLP";PUBMED_ID=17012160;trait_ID=2234;
# trait="Spleen percentage";breed="leghorn";"FlankMarkers=ADL0022";VTO_name="spleen mass";
# CMO_name="spleen weight to body weight ratio";Map_Type="Linkage";Model="Mendelian";
# Test_Base="Chromosome-wise";Significance="Significant";P-value="<0.05";F-Stat="5.52";
# Variance="2.94";Dominance_Effect="-0.002";Additive_Effect="0.01"
# make dictionary of attributes
# keys are:
# QTL_ID,Name,Abbrev,PUBMED_ID,trait_ID,trait,FlankMarkers,
# VTO_name,Map_Type,Significance,P-value,Model,
# Test_Base,Variance, Bayes-value,PTO_name,gene_IDsrc,peak_cM,
# CMO_name,gene_ID,F-Stat,LOD-score,Additive_Effect,
# Dominance_Effect,Likelihood_Ratio,LS-means,Breed,
# trait (duplicate with Name),Variance,Bayes-value,
# F-Stat,LOD-score,Additive_Effect,Dominance_Effect,
# Likelihood_Ratio,LS-means
# deal with poorly formed attributes
if re.search(r'"FlankMarkers";', attr):
attr = re.sub(r'FlankMarkers;', '', attr)
attr_items = re.sub(r'"', '', attr).split(";")
bad_attrs = set()
for a in attr_items:
if not re.search(r'=', a):
# bad_attr_flag = True # TODO unused
# remove this attribute from the list
bad_attrs.add(a)
attr_set = set(attr_items) - bad_attrs
attribute_dict = dict(item.split("=") for item in attr_set)
qtl_num = attribute_dict.get('QTL_ID')
if self.testMode and int(qtl_num) not in self.test_ids:
continue
# make association between QTL and trait
qtl_id = 'AQTL:' + str(qtl_num)
model.addIndividualToGraph(qtl_id, None, geno.genoparts['QTL'])
geno.addTaxon(taxon_id, qtl_id)
trait_id = 'AQTLTrait:'+attribute_dict.get('trait_ID')
# if pub is in attributes, add it to the association
pub_id = None
if 'PUBMED_ID' in attribute_dict.keys():
pub_id = attribute_dict.get('PUBMED_ID')
if re.match(r'ISU.*', pub_id):
pub_id = 'AQTLPub:' + pub_id.strip()
reference = Reference(g, pub_id)
else:
pub_id = 'PMID:' + pub_id.strip()
reference = Reference(
g, pub_id, Reference.ref_types['journal_article'])
reference.addRefToGraph()
# Add QTL to graph
assoc = G2PAssoc(
g, self.name, qtl_id, trait_id,
model.object_properties['is_marker_for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
if 'P-value' in attribute_dict.keys():
s = re.sub(r'<', '', attribute_dict.get('P-value'))
if ',' in s:
s = re.sub(r',', '.', s)
if s.isnumeric():
score = float(s)
assoc.set_score(score)
assoc.add_association_to_graph()
# TODO make association to breed
# (which means making QTL feature in Breed background)
# get location of QTL
chromosome = re.sub(r'Chr\.', '', chromosome)
chrom_id = makeChromID(chromosome, taxon_id, 'CHR')
chrom_in_build_id = \
makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(
chromosome, build_id, build_label, chrom_id)
qtl_feature = Feature(g, qtl_id, None, geno.genoparts['QTL'])
if start_bp == '':
start_bp = None
qtl_feature.addFeatureStartLocation(
start_bp, chrom_in_build_id, strand,
[Feature.types['FuzzyPosition']])
if stop_bp == '':
stop_bp = None
qtl_feature.addFeatureEndLocation(
stop_bp, chrom_in_build_id, strand,
[Feature.types['FuzzyPosition']])
qtl_feature.addTaxonToFeature(taxon_id)
qtl_feature.addFeatureToGraph()
if not self.testMode and \
limit is not None and line_counter > limit:
break
logger.warning("Bad attribute flags in this file")
logger.info("Done with QTL genomic mappings for %s", taxon_id)
return
def _process_QTLs_genomic_location(self, raw, taxon_id, build_id, build_label, limit=None):
"""
This method
Triples created:
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
gu = GraphUtils(curie_map.get())
line_counter = 0
geno = Genotype(g)
genome_id = geno.makeGenomeID(taxon_id) # assume that chrs get added to the genome elsewhere
eco_id = "ECO:0000061" # Quantitative Trait Analysis Evidence
with gzip.open(raw, 'rt', encoding='ISO-8859-1') as tsvfile:
reader = csv.reader(tsvfile, delimiter="\t")
for row in reader:
line_counter += 1
if re.match('^#', ' '.join(row)):
continue
(chromosome, qtl_source, qtl_type, start_bp, stop_bp, frame, strand, score, attr) = row
# Chr.Z Animal QTLdb Production_QTL 33954873 34023581 . . .
# QTL_ID=2242;Name="Spleen percentage";Abbrev="SPLP";PUBMED_ID=17012160;trait_ID=2234;
# trait="Spleen percentage";breed="leghorn";"FlankMarkers=ADL0022";VTO_name="spleen mass";
# CMO_name="spleen weight to body weight ratio";Map_Type="Linkage";Model="Mendelian";
# Test_Base="Chromosome-wise";Significance="Significant";P-value="<0.05";F-Stat="5.52";
# Variance="2.94";Dominance_Effect="-0.002";Additive_Effect="0.01"
# make dictionary of attributes
# keys are:
# QTL_ID,Name,Abbrev,PUBMED_ID,trait_ID,trait,
# FlankMarkers,VTO_name,Map_Type,Significance,P-value,Model,Test_Base,Variance,
# Bayes-value,PTO_name,gene_IDsrc,peak_cM,CMO_name,gene_ID,F-Stat,LOD-score,Additive_Effect,
# Dominance_Effect,Likelihood_Ratio,LS-means,Breed,
# trait (duplicate with Name),Variance,Bayes-value,
# F-Stat,LOD-score,Additive_Effect,Dominance_Effect,Likelihood_Ratio,LS-means
# deal with poorly formed attributes
if re.search('"FlankMarkers";', attr):
attr = re.sub('"FlankMarkers";', '', attr)
attr_items = re.sub('"', '', attr).split(";")
bad_attr_flag = False
for a in attr_items:
if not re.search('=', a):
bad_attr_flag = True
if bad_attr_flag:
logger.error("Poorly formed data on line %d:\n %s", line_counter, '\t'.join(row))
continue
attribute_dict = dict(item.split("=") for item in re.sub('"', '', attr).split(";"))
qtl_num = attribute_dict.get('QTL_ID')
if self.testMode and int(qtl_num) not in self.test_ids:
continue
# make association between QTL and trait
qtl_id = 'AQTL:' + str(qtl_num)
gu.addIndividualToGraph(g, qtl_id, None, geno.genoparts['QTL'])
geno.addTaxon(taxon_id, qtl_id)
trait_id = 'AQTLTrait:'+attribute_dict.get('trait_ID')
# if pub is in attributes, add it to the association
pub_id = None
if 'PUBMED_ID' in attribute_dict.keys():
pub_id = attribute_dict.get('PUBMED_ID')
if re.match('ISU.*', pub_id):
pub_id = 'AQTLPub:' + pub_id.strip()
p = Reference(pub_id)
else:
pub_id = 'PMID:' + pub_id.strip()
p = Reference(pub_id, Reference.ref_types['journal_article'])
p.addRefToGraph(g)
# Add QTL to graph
assoc = G2PAssoc(self.name, qtl_id, trait_id, gu.object_properties['is_marker_for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
if 'P-value' in attribute_dict.keys():
score = float(re.sub('<', '', attribute_dict.get('P-value')))
assoc.set_score(score)
assoc.add_association_to_graph(g)
# TODO make association to breed (which means making QTL feature in Breed background)
# get location of QTL
chromosome = re.sub('Chr\.', '', chromosome)
chrom_id = makeChromID(chromosome, taxon_id, 'CHR')
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(chromosome, build_id, build_label, chrom_id)
qtl_feature = Feature(qtl_id, None, geno.genoparts['QTL'])
if start_bp == '':
start_bp = None
qtl_feature.addFeatureStartLocation(start_bp, chrom_in_build_id, strand,
[Feature.types['FuzzyPosition']])
if stop_bp == '':
stop_bp = None
qtl_feature.addFeatureEndLocation(stop_bp, chrom_in_build_id, strand,
[Feature.types['FuzzyPosition']])
qtl_feature.addTaxonToFeature(g, taxon_id)
qtl_feature.addFeatureToGraph(g)
if not self.testMode and limit is not None and line_counter > limit:
break
logger.info("Done with QTL genomic mappings for %s", taxon_id)
return
def _process_QTLs_genetic_location(self, raw, taxon_id, common_name, limit=None):
"""
This function processes
Triples created:
:param limit:
:return:
"""
if self.testMode:
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu = GraphUtils(curie_map.get())
eco_id = "ECO:0000061" # Quantitative Trait Analysis Evidence
logger.info("Processing genetic location for %s", taxon_id)
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
for row in filereader:
line_counter += 1
(qtl_id, qtl_symbol, trait_name, assotype, empty, chromosome, position_cm, range_cm,
flankmark_a2, flankmark_a1, peak_mark, flankmark_b1, flankmark_b2, exp_id, model, test_base,
sig_level, lod_score, ls_mean, p_values, f_statistics, variance, bayes_value, likelihood_ratio,
trait_id, dom_effect, add_effect, pubmed_id, gene_id, gene_id_src, gene_id_type, empty2) = row
if self.testMode and int(qtl_id) not in self.test_ids:
continue
qtl_id = 'AQTL:'+qtl_id
trait_id = 'AQTLTrait:'+trait_id
# Add QTL to graph
f = Feature(qtl_id, qtl_symbol, geno.genoparts['QTL'])
f.addTaxonToFeature(g, taxon_id)
# deal with the chromosome
chrom_id = makeChromID(chromosome, taxon_id, 'CHR')
# add a version of the chromosome which is defined as the genetic map
build_id = 'MONARCH:'+common_name.strip()+'-linkage'
build_label = common_name+' genetic map'
geno.addReferenceGenome(build_id, build_label, taxon_id)
chrom_in_build_id = makeChromID(chromosome, build_id, 'MONARCH')
geno.addChromosomeInstance(chromosome, build_id, build_label, chrom_id)
start = stop = None
if re.search('-', range_cm):
range_parts = re.split('-', range_cm)
# check for poorly formed ranges
if len(range_parts) == 2 and range_parts[0] != '' and range_parts[1] != '':
(start, stop) = [int(float(x.strip())) for x in re.split('-', range_cm)]
else:
logger.info("There's a cM range we can't handle for QTL %s: %s", qtl_id, range_cm)
elif position_cm != '':
start = stop = int(float(position_cm))
# FIXME remove converion to int for start/stop when schema can handle floats
# add in the genetic location based on the range
f.addFeatureStartLocation(start, chrom_in_build_id, None, [Feature.types['FuzzyPosition']])
f.addFeatureEndLocation(stop, chrom_in_build_id, None, [Feature.types['FuzzyPosition']])
f.addFeatureToGraph(g)
# sometimes there's a peak marker, like a rsid. we want to add that as a variant of the gene,
# and xref it to the qtl.
dbsnp_id = None
if peak_mark != '' and peak_mark != '.' and re.match('rs', peak_mark.strip()):
dbsnp_id = 'dbSNP:'+peak_mark.strip()
gu.addIndividualToGraph(g, dbsnp_id, None, geno.genoparts['sequence_alteration'])
gu.addXref(g, qtl_id, dbsnp_id)
if gene_id is not None and gene_id != '' and gene_id != '.':
if gene_id_src == 'NCBIgene' or gene_id_src == '': # we assume if no src is provided, it's NCBI
gene_id = 'NCBIGene:'+gene_id.strip()
geno.addGene(gene_id, None) # we will expect that these labels provided elsewhere
geno.addAlleleOfGene(qtl_id, gene_id, geno.object_properties['feature_to_gene_relation']) # FIXME what is the right relationship here?
if dbsnp_id is not None:
# add the rsid as a seq alt of the gene_id
vl_id = '_' + re.sub(':', '', gene_id) + '-' + peak_mark
if self.nobnodes:
vl_id = ':' + vl_id
geno.addSequenceAlterationToVariantLocus(dbsnp_id, vl_id)
geno.addAlleleOfGene(vl_id, gene_id)
# add the trait
gu.addClassToGraph(g, trait_id, trait_name)
# Add publication
r = None
if re.match('ISU.*', pubmed_id):
pub_id = 'AQTLPub:'+pubmed_id.strip()
r = Reference(pub_id)
elif pubmed_id != '':
pub_id = 'PMID:'+pubmed_id.strip()
r = Reference(pub_id, Reference.ref_types['journal_article'])
if r is not None:
r.addRefToGraph(g)
# make the association to the QTL
assoc = G2PAssoc(self.name, qtl_id, trait_id, gu.object_properties['is_marker_for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id as evidence
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
score = float(re.sub('<', '', p_values))
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph(g)
# make the association to the dbsnp_id, if found
if dbsnp_id is not None:
# make the association to the dbsnp_id
assoc = G2PAssoc(self.name, dbsnp_id, trait_id, gu.object_properties['is_marker_for'])
assoc.add_evidence(eco_id)
assoc.add_source(pub_id)
# create a description from the contents of the file
# desc = ''
# assoc.addDescription(g, assoc_id, desc)
# TODO add exp_id
# if exp_id != '':
# exp_id = 'AQTLExp:'+exp_id
# gu.addIndividualToGraph(g, exp_id, None, eco_id)
if p_values != '':
score = float(re.sub('<', '', p_values))
assoc.set_score(score) # todo add score type
# TODO add LOD score?
assoc.add_association_to_graph(g)
if not self.testMode and limit is not None and line_counter > limit:
break
logger.info("Done with QTL genetic info")
return
def process_catalog(self, limit=None):
"""
:param limit:
:return:
"""
raw = '/'.join((self.rawdir, self.files['catalog']['file']))
logger.info("Processing Data from %s", raw)
gu = GraphUtils(curie_map.get())
if self.testMode: # set the graph to build
g = self.testgraph
else:
g = self.graph
line_counter = 0
geno = Genotype(g)
gu.loadProperties(g, geno.object_properties, gu.OBJPROP)
gu.loadAllProperties(g)
tax_id = 'NCBITaxon:9606' # hardcode
genome_version = 'GRCh38' # hardcode
# build a hashmap of genomic location to identifiers,
# to try to get the equivalences
loc_to_id_hash = {}
with open(raw, 'r', encoding="iso-8859-1") as csvfile:
filereader = csv.reader(csvfile, delimiter='\t', quotechar='\"')
next(filereader, None) # skip the header row
for row in filereader:
if not row:
pass
else:
line_counter += 1
(date_added_to_catalog, pubmed_num, first_author, pub_date,
journal, link, study_name, disease_or_trait,
initial_sample_description, replicate_sample_description,
region, chrom_num, chrom_pos, reported_gene_nums,
mapped_gene, upstream_gene_num, downstream_gene_num,
snp_gene_nums, upstream_gene_distance,
downstream_gene_distance, strongest_snp_risk_allele, snps,
merged, snp_id_current, context, intergenic_flag,
risk_allele_frequency, pvalue, pvalue_mlog, pvalue_text,
or_or_beta, confidence_interval_95,
platform_with_snps_passing_qc, cnv_flag, mapped_trait,
mapped_trait_uri) = row
intersect = \
list(set([str(i) for i in self.test_ids['gene']]) &
set(re.split(r',', snp_gene_nums)))
# skip if no matches found in test set
if self.testMode and len(intersect) == 0:
continue
# 06-May-2015 25917933 Zai CC 20-Nov-2014 J Psychiatr Res http://europepmc.org/abstract/MED/25917933
# A genome-wide association study of suicide severity scores in bipolar disorder.
# Suicide in bipolar disorder
# 959 European ancestry individuals NA
# 10p11.22 10 32704340 C10orf68, CCDC7, ITGB1 CCDC7
# rs7079041-A rs7079041 0 7079041 intron 0 2E-6 5.698970
if chrom_num != '' and chrom_pos != '':
loc = 'chr'+str(chrom_num)+':'+str(chrom_pos)
if loc not in loc_to_id_hash:
loc_to_id_hash[loc] = set()
else:
loc = None
if re.search(r' x ', strongest_snp_risk_allele) \
or re.search(r',', strongest_snp_risk_allele):
# TODO deal with haplotypes
logger.warning(
"We can't deal with haplotypes yet: %s",
strongest_snp_risk_allele)
continue
elif re.match(r'rs', strongest_snp_risk_allele):
rs_id = 'dbSNP:'+strongest_snp_risk_allele.strip()
# remove the alteration
elif re.match(r'kgp', strongest_snp_risk_allele):
# FIXME this isn't correct
rs_id = 'dbSNP:'+strongest_snp_risk_allele.strip()
# http://www.1000genomes.org/faq/what-are-kgp-identifiers
# for some information
# They were created by Illumina for their genotyping
# platform before some variants identified during the
# pilot phase of the project had been assigned
# rs numbers.
elif re.match(r'chr', strongest_snp_risk_allele):
# like: chr10:106180121-G
rs_id = ':gwas-' + \
re.sub(
r':', '-', strongest_snp_risk_allele.strip())
elif strongest_snp_risk_allele.strip() == '':
# logger.debug(
# "No strongest SNP risk allele for %s:\n%s",
# pubmed_num, str(row))
# FIXME still consider adding in the EFO terms
# for what the study measured?
continue
else:
logger.warning(
"There's a snp id i can't manage: %s",
strongest_snp_risk_allele)
continue
alteration = re.search(r'-(.*)$', rs_id)
if alteration is not None \
and re.match(r'[ATGC]', alteration.group(1)):
# add variation to snp
pass # TODO
rs_id = re.sub(r'-.*$', '', rs_id).strip()
if loc is not None:
loc_to_id_hash[loc].add(rs_id)
pubmed_id = 'PMID:'+pubmed_num
r = Reference(
pubmed_id, Reference.ref_types['journal_article'])
r.addRefToGraph(g)
# create the chromosome
chrom_id = makeChromID(chrom_num, genome_version, 'CHR')
# add the feature to the graph
snp_description = None
if risk_allele_frequency != '' and \
risk_allele_frequency != 'NR':
snp_description = \
str(risk_allele_frequency) + \
' [risk allele frequency]'
f = Feature(
rs_id, strongest_snp_risk_allele.strip(),
Feature.types[r'SNP'], snp_description)
if chrom_num != '' and chrom_pos != '':
f.addFeatureStartLocation(chrom_pos, chrom_id)
f.addFeatureEndLocation(chrom_pos, chrom_id)
f.addFeatureToGraph(g)
f.addTaxonToFeature(g, tax_id)
# TODO consider adding allele frequency as property;
# but would need background info to do that
# also want to add other descriptive info about
# the variant from the context
for c in re.split(r';', context):
cid = self._map_variant_type(c.strip())
if cid is not None:
gu.addType(g, rs_id, cid)
# add deprecation information
if merged == 1 and str(snp_id_current.strip()) != '':
# get the current rs_id
current_rs_id = 'dbSNP:'
if not re.match(r'rs', snp_id_current):
current_rs_id += 'rs'
if loc is not None:
loc_to_id_hash[loc].append(current_rs_id)
current_rs_id += str(snp_id_current)
gu.addDeprecatedIndividual(g, rs_id, current_rs_id)
# TODO check on this
# should we add the annotations to the current
# or orig?
gu.makeLeader(g, current_rs_id)
else:
gu.makeLeader(g, rs_id)
# add the feature as a sequence alteration
# affecting various genes
# note that intronic variations don't necessarily list
# the genes such as for rs10448080 FIXME
if snp_gene_nums != '':
for s in re.split(r',', snp_gene_nums):
s = s.strip()
# still have to test for this,
# because sometimes there's a leading comma
if s != '':
gene_id = 'NCBIGene:'+s
geno.addAlleleOfGene(rs_id, gene_id)
# add the up and downstream genes if they are available
if upstream_gene_num != '':
downstream_gene_id = 'NCBIGene:'+downstream_gene_num
gu.addTriple(
g, rs_id,
Feature.object_properties[
r'upstream_of_sequence_of'],
downstream_gene_id)
if downstream_gene_num != '':
upstream_gene_id = 'NCBIGene:'+upstream_gene_num
gu.addTriple(
g, rs_id,
Feature.object_properties[
'downstream_of_sequence_of'],
upstream_gene_id)
description = 'A study of ' + disease_or_trait + \
' in ' + initial_sample_description
if replicate_sample_description != '':
description = \
' '.join(
(description, 'with',
replicate_sample_description))
if platform_with_snps_passing_qc != '':
description = ' '.join(
(description, 'on platform',
platform_with_snps_passing_qc))
description = ' '.join((description, '(p='+pvalue+')'))
# make associations to the EFO terms; there can be >1
if mapped_trait_uri.strip() != '':
for t in re.split(r',', mapped_trait_uri):
t = t.strip()
cu = CurieUtil(curie_map.get())
tid = cu.get_curie(t)
assoc = G2PAssoc(
self.name, rs_id, tid,
gu.object_properties['contributes_to'])
assoc.add_source(pubmed_id)
# combinatorial evidence
# used in automatic assertion
eco_id = 'ECO:0000213'
assoc.add_evidence(eco_id)
# assoc.set_description(description)
# FIXME score should get added to provenance/study
# assoc.set_score(pvalue)
assoc.add_association_to_graph(g)
if not self.testMode and\
(limit is not None and line_counter > limit):
break
Assoc(self.name).load_all_properties(g)
# loop through the location hash,
# and make all snps at that location equivalent
for l in loc_to_id_hash:
snp_ids = loc_to_id_hash[l]
if len(snp_ids) > 1:
logger.info("%s has >1 snp id: %s", l, str(snp_ids))
return
