def show_es_gene_section(gene_symbol=None,
seqid=None,
start_pos=None,
end_pos=None):
''' Template inclusion tag to render a gene section given a
chado gene feature. '''
if seqid is not None and isinstance(seqid,
str) and seqid.startswith("chr"):
seqid = seqid
else:
seqid = 'chr' + str(seqid)
if gene_symbol is not None:
''' gene symbol query'''
query = ElasticQuery.query_match("gene_symbol", gene_symbol)
elif end_pos is None:
''' start and end are same, range query for snp'''
query_bool = BoolQuery(must_arr=[
Query.match("seqid", seqid),
RangeQuery("featureloc.start", lte=start_pos),
RangeQuery("featureloc.end", gte=start_pos)
])
query = ElasticQuery.bool(query_bool)
else:
''' start and end are same, range query for snp'''
query_bool = BoolQuery(must_arr=[
Query.match("seqid", seqid),
RangeQuery("featureloc.start", gte=start_pos),
RangeQuery("featureloc.end", lte=end_pos)
])
query = ElasticQuery.bool(query_bool)
elastic = Search(query, idx=ElasticSettings.idx(name='GENE'))
return {'es_genes': elastic.search().docs}
def show_es_gene_section(gene_symbol=None,
seqid=None,
start_pos=None,
end_pos=None):
''' Template inclusion tag to render a gene section given a
chado gene feature. '''
seqid = str(seqid).replace('chr', '')
if gene_symbol is not None:
''' gene symbol query'''
query = ElasticQuery.query_match("symbol", gene_symbol)
elif end_pos is None:
''' start and end are same, range query for snp'''
query_bool = BoolQuery(must_arr=[
Query.match("chromosome", seqid),
RangeQuery("start", lte=start_pos),
RangeQuery("stop", gte=start_pos)
])
query = ElasticQuery.bool(query_bool)
else:
''' start and end are same, range query for snp'''
query_bool = BoolQuery(must_arr=[
Query.match("chromosome", seqid),
RangeQuery("start", gte=start_pos),
RangeQuery("stop", lte=end_pos)
])
query = ElasticQuery.bool(query_bool)
elastic = Search(query, idx=ElasticSettings.idx(name='GENE'))
return {'es_genes': elastic.search().docs}
def test_bool_query(self):
''' Test a bool query. '''
query_bool = BoolQuery()
highlight = Highlight(["id", "seqid"])
query_bool.must(Query.term("id", "rs768019142")) \
.must(RangeQuery("start", gt=1000)) \
.must_not(Query.match("seqid", "2")) \
.should(Query.match("seqid", "3")) \
.should(Query.match("seqid", "1"))
query = ElasticQuery.bool(query_bool, highlight=highlight)
elastic = Search(query, idx=ElasticSettings.idx('DEFAULT'))
self.assertTrue(len(elastic.search().docs) == 1, "Elastic string query retrieved marker (rs768019142)")
def test_bool_nested_filter(self):
''' Test combined Bool filter '''
query_bool_nest = BoolQuery()
query_bool_nest.must(Query.match("id", "rs768019142").query_wrap()) \
.must(Query.term("seqid", 1))
query_bool = BoolQuery()
query_bool.should(query_bool_nest) \
.should(Query.term("seqid", 2))
query = ElasticQuery.filtered_bool(Query.match_all(), query_bool, sources=["id", "seqid", "start"])
elastic = Search(query, idx=ElasticSettings.idx('DEFAULT'))
self.assertTrue(elastic.search().hits_total >= 1, "Nested bool filter query")
def test_bool_filtered_query4(self):
''' Test building and running a filtered boolean query.
Note: ElasticQuery used to wrap match in a query object. '''
query_bool = BoolQuery()
query_bool.should(RangeQuery("start", lte=20000)) \
.should(Query.term("seqid", 2)) \
.must(Query.match("id", "rs768019142").query_wrap()) \
.must(Query.term("seqid", 1))
query = ElasticQuery.filtered_bool(Query.match_all(), query_bool, sources=["id", "seqid", "start"])
elastic = Search(query, idx=ElasticSettings.idx('DEFAULT'))
self.assertTrue(elastic.search().hits_total == 1, "Elastic filtered query retrieved marker (rs768019142)")
def _get_current_build_info(self, seqid, position):
''' Get upper & lower boundaries for a hit given the position of the marker.'''
query = ElasticQuery(
BoolQuery(must_arr=[
RangeQuery("position", gte=position),
Query.match("seqid", seqid)
]))
result = Search(query,
idx=ElasticSettings.idx('HAPMAP', 'HAPMAP'),
qsort=Sort('position:asc'),
size=1).search()
genetic_map_position = getattr(result.docs[0], "genetic_map_position")
query = ElasticQuery(
BoolQuery(must_arr=[
RangeQuery("genetic_map_position",
gte=(genetic_map_position + 0.1)),
Query.match("seqid", seqid)
]))
result = Search(query,
idx=ElasticSettings.idx('HAPMAP', 'HAPMAP'),
qsort=Sort('position:asc'),
size=1).search()
start = int(getattr(result.docs[0], "position"))
query = ElasticQuery(
BoolQuery(must_arr=[
RangeQuery("genetic_map_position",
lte=(genetic_map_position - 0.1)),
Query.match("seqid", seqid)
]))
result = Search(query,
idx=ElasticSettings.idx('HAPMAP', 'HAPMAP'),
qsort=Sort('position:desc'),
size=1).search()
end = int(getattr(result.docs[0], "position"))
build_info = {'build': 38, 'seqid': seqid, 'start': start, 'end': end}
return build_info
def test_mapping_parent_child(self):
''' Test creating mapping with parent child relationship. '''
gene_mapping = MappingProperties("gene")
gene_mapping.add_property("symbol", "string", analyzer="full_name")
inta_mapping = MappingProperties("publication", "gene")
load = Loader()
idx = "test__mapping__"+SEARCH_SUFFIX
options = {"indexName": idx, "shards": 1}
requests.delete(ElasticSettings.url() + '/' + idx)
# add child mappings first
status = load.mapping(inta_mapping, "publication", analyzer=Loader.KEYWORD_ANALYZER, **options)
self.assertTrue(status, "mapping inteactions")
status = load.mapping(gene_mapping, "gene", analyzer=Loader.KEYWORD_ANALYZER, **options)
self.assertTrue(status, "mapping genes")
''' load docs and test has parent query'''
json_data = '{"index": {"_index": "%s", "_type": "gene", "_id" : "1"}}\n' % idx
json_data += json.dumps({"symbol": "PAX1"}) + '\n'
json_data += '{"index": {"_index": "%s", "_type": "publication", "_id" : "2", "parent": "1"}}\n' % idx
json_data += json.dumps({"pubmed": 1234}) + '\n'
Bulk.load(idx, '', json_data)
Search.index_refresh(idx)
query = ElasticQuery.has_parent('gene', Query.match('symbol', 'PAX1'))
elastic = Search(query, idx=idx, idx_type='publication', size=500)
docs = elastic.search().docs
self.assertEquals(len(docs), 1)
self.assertEquals(getattr(docs[0], 'pubmed'), 1234)
self.assertEquals(docs[0].parent(), '1')
self.assertRaises(QueryError, ElasticQuery.has_parent, 'gene', 'xxxxx')
''' test has child query '''
query = ElasticQuery.has_child('publication', Query.match('pubmed', 1234))
elastic = Search(query, idx=idx, idx_type='gene', size=500)
docs = elastic.search().docs
self.assertEquals(len(docs), 1)
self.assertEquals(getattr(docs[0], 'symbol'), 'PAX1')
self.assertEquals(docs[0].parent(), None)
requests.delete(ElasticSettings.url() + '/' + idx)
def show_es_gene_section(gene_symbol=None, seqid=None,
start_pos=None, end_pos=None):
''' Template inclusion tag to render a gene section given a
chado gene feature. '''
seqid = str(seqid).replace('chr', '')
if gene_symbol is not None:
''' gene symbol query'''
query = ElasticQuery.query_match("symbol", gene_symbol)
elif end_pos is None:
''' start and end are same, range query for snp'''
query_bool = BoolQuery(must_arr=[Query.match("chromosome", seqid),
RangeQuery("start", lte=start_pos),
RangeQuery("stop", gte=start_pos)])
query = ElasticQuery.bool(query_bool)
else:
''' start and end are same, range query for snp'''
query_bool = BoolQuery(must_arr=[Query.match("chromosome", seqid),
RangeQuery("start", gte=start_pos),
RangeQuery("stop", lte=end_pos)])
query = ElasticQuery.bool(query_bool)
elastic = Search(query, idx=ElasticSettings.idx(name='GENE'))
return {'es_genes': elastic.search().docs}
def query_match(cls, match_id, match_str, sources=None, highlight=None):
''' Factory method for creating elastic Match Query.
@type match_id: string
@param match_id: The match id.
@type match_str: string
@param match_str: The string value to match.
@type sources: array of result fields
@keyword sources: The _source filtering to be used (default: None).
@type highlight: Highlight
@keyword highlight: Define the highlighting of results (default: None).
@return: L{ElasticQuery}
'''
query = Query.match(match_id, match_str)
return cls(query, sources, highlight)
def _get_chr_band(self, seqid, position):
''' Get chr band for a given chr/position '''
if seqid == 6 and position >= 24891793 and position <= 34924245:
return 'MHC'
query = ElasticQuery(
BoolQuery(must_arr=[
Query.match("seqid", seqid),
RangeQuery("start", lte=position),
RangeQuery("stop", gte=position)
]))
result = Search(query, idx=ElasticSettings.idx('BAND', 'BAND'),
size=1).search()
return (getattr(result.docs[0], "seqid") +
getattr(result.docs[0], "name"))
def pad_region_doc(cls, region):
'''Adds details of disease_loci & hits for a given region doc'''
hits_idx = ElasticSettings.idx('REGION', 'STUDY_HITS')
disease_loci = getattr(region, "disease_loci")
locus_start = Agg('region_start', 'min', {'field': 'build_info.start'})
locus_end = Agg('region_end', 'max', {'field': 'build_info.end'})
match_agg = Agg('filtered_result', 'filter', Query.match("build_info.build", 38).query_wrap(),
sub_agg=[locus_start, locus_end])
build_info_agg = Agg('build_info', 'nested', {"path": 'build_info'}, sub_agg=[match_agg])
query = ElasticQuery(FilteredQuery(Query.terms("disease_locus", disease_loci),
Filter(BoolQuery(should_arr=[Query.missing_terms("field", "group_name")]
))))
resultObj = Search(search_query=query, idx=hits_idx, aggs=Aggs(build_info_agg)).search()
hit_ids = []
markers = []
genes = []
studies = []
pmids = []
for doc in resultObj.docs:
hit_ids.append(doc.doc_id())
markers.append(getattr(doc, "marker"))
if hasattr(doc, "genes") and getattr(doc, "genes") != None:
genes.extend([g for g in getattr(doc, "genes")])
studies.append(getattr(doc, "dil_study_id"))
pmids.append(getattr(doc, "pmid"))
build_info = getattr(resultObj.aggs['build_info'], 'filtered_result')
region_start = int(build_info['region_start']['value'])
region_end = int(build_info['region_end']['value'])
build_info = {
'build': 38,
'seqid': getattr(region, "seqid"),
'start': region_start,
'end': region_end
}
setattr(region, "build_info", build_info)
setattr(region, "hits", hit_ids)
setattr(region, "markers", list(set(markers)))
setattr(region, "genes", list(set(genes)))
setattr(region, "studies", list(set(studies)))
setattr(region, "pmids", list(set(pmids)))
return region
def chicpeaSearch(request, url):
queryDict = request.GET
user = request.user
targetIdx = queryDict.get("targetIdx")
blueprint = {}
hic = []
addList = []
searchType = 'gene'
searchTerm = queryDict.get("searchTerm").upper()
searchTerm = searchTerm.replace(",", "")
searchTerm = searchTerm.replace("..", "-")
searchTerm = searchTerm.replace(" ", "") # Chris suggestion to prevent issue with spaces in queries
snpTrack = queryDict.get("snp_track")
(idx_keys_auth, idx_type_keys_auth) = get_authenticated_idx_and_idx_types(
user=user, idx_keys=None, idx_type_keys=None)
if snpTrack:
mo = re.match(r"(.*)-(.*)", snpTrack)
(group, track) = mo.group(1, 2) # @UnusedVariable
if group != 'ud' and 'CP_STATS_'+group.upper()+'.'+snpTrack.upper() not in idx_type_keys_auth:
snpTrack = None
if targetIdx not in utils.tissues:
for target in getattr(chicp_settings, 'CP_TARGET'):
if 'CP_TARGET_'+target not in idx_keys_auth:
if targetIdx == target:
retJSON = {'error': 'Sorry, you do not have permission to view this dataset.'}
return JsonResponse(retJSON)
continue
elasticJSON = Search(idx=ElasticSettings.idx('CP_TARGET_'+target)).get_mapping(mapping_type="gene_target")
tissueList = list(elasticJSON[ElasticSettings.idx('CP_TARGET_'+target)]
['mappings']['gene_target']['_meta']['tissue_type'].keys())
utils.tissues['CP_TARGET_'+target] = tissueList
if queryDict.get("region") or re.match(r"(.*):(\d+)-(\d+)", searchTerm):
searchType = 'region'
region = searchTerm
if queryDict.get("region"):
region = queryDict.get("region")
else:
searchTerm = ""
mo = re.match(r"(.*):(\d+)-(\d+)", region)
(chrom, segmin, segmax) = mo.group(1, 2, 3)
chrom = chrom.replace('chr', "")
chrom = chrom.replace('CHR', "")
if re.search("^rs[0-9]+", searchTerm.lower()):
searchTerm = searchTerm.lower()
addList.append(_find_snp_position(snpTrack, searchTerm))
if addList[0].get("error"):
return JsonResponse({'error': addList[0]['error']})
position = addList[0]['end']
if searchType != 'region':
searchType = 'snp'
logger.warn("### "+searchType+" - "+searchTerm+' ###')
if searchType == 'region':
query_bool = BoolQuery()
filter_bool = BoolQuery()
if searchTerm and len(addList) == 0 and re.match(r"(.*):(\d+)-(\d+)",
queryDict.get("searchTerm").replace(",", "")) == None:
query_bool.must([Query.query_string(searchTerm, fields=["name", "ensg"]),
Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
else:
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
if len(addList) > 0:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
else:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", gte=segmin, lte=segmax),
RangeQuery("baitEnd", gte=segmin, lte=segmax)]),
BoolQuery(must_arr=[RangeQuery("oeStart", gte=segmin, lte=segmax),
RangeQuery("oeEnd", gte=segmin, lte=segmax)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, v1, v2) = _build_hic_query(query, targetIdx, segmin, segmax) # @UnusedVariable
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': queryDict.get("searchTerm")+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
elif searchType == 'snp':
if len(addList) > 0:
chrom = addList[0]['chr']
query_bool = BoolQuery()
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
filter_bool = BoolQuery()
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
hic, segmin, segmax = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Marker '+searchTerm+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
else:
# geneQuery = ElasticQuery.query_string(searchTerm, fields=["gene_name"])
geneQuery = ElasticQuery.filtered(Query.match_all(), Filter(Query.match("gene_name", searchTerm).query_wrap()))
resultObj = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/',
search_query=geneQuery, size=0, qsort=Sort('seqid:asc,start')).search()
if resultObj.hits_total > 1:
geneResults = []
resultObj2 = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/', search_query=geneQuery,
size=(resultObj.hits_total+1), qsort=Sort('seqid:asc,start')).search()
docs = resultObj2.docs
gene_ids = [getattr(doc, 'attr')['gene_id'][1:-1] for doc in docs]
query = ElasticQuery.filtered(Query.match_all(), TermsFilter.get_terms_filter('ensg', gene_ids))
agg = Agg('ensg_agg', "terms", {"field": "ensg", "size": 0})
res = Search(idx=ElasticSettings.idx('CP_TARGET_'+targetIdx), search_query=query, aggs=Aggs(agg),
size=0).search()
ensg_count = res.aggs['ensg_agg'].get_buckets()
gene_ids = [g['key'] for g in ensg_count]
for d in resultObj2.docs:
if getattr(d, "attr")["gene_id"].replace('\"', '') in gene_ids:
geneResults.append({
'gene_name': getattr(d, "attr")["gene_name"].replace('\"', ''),
'gene_id': getattr(d, "attr")["gene_id"].replace('\"', ''),
'location': "chr" + getattr(d, "seqid") + ":" +
locale.format_string("%d", getattr(d, "start"), grouping=True) + ".." +
locale.format_string("%d", getattr(d, "end"), grouping=True),
})
if len(geneResults) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
elif len(geneResults) > 1:
retJSON = {
'error': 'Gene name <strong>'+searchTerm+'</strong> returns too many hits, please select your prefered result from the list below.',
'results': geneResults,
'cols': ['HGNC Symbol', 'Ensembl Gene ID', 'Location']
}
return JsonResponse(retJSON)
query_bool = BoolQuery()
query_bool.must([RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
query = ElasticQuery.filtered_bool(Query.query_string(searchTerm, fields=["name", "ensg", "oeName"]),
query_bool, sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, segmin, segmax) = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
chrom = hic[0]['baitChr']
try:
chrom
except NameError:
retJSON = {'error': 'No chromosome defined for search'}
return JsonResponse(retJSON)
# get genes based on this segment
genes = _build_gene_query(chrom, segmin, segmax)
(snps, snpMeta) = _build_snp_query(snpTrack, chrom, segmin, segmax)
frags = _build_frags_query(getattr(chicp_settings, 'DEFAULT_FRAG'), chrom, segmin, segmax)
addList = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], addList)
retJSON = {"hic": hic,
"frags": frags,
"meta": {"ostart": int(segmin),
"oend": int(segmax),
"rstart": 1,
"rend": int(segmax) - int(segmin),
"rchr": str(chrom),
"tissues": utils.tissues['CP_TARGET_'+targetIdx]},
"snps": snps,
"snp_meta": snpMeta,
"genes": genes,
"region": str(chrom) + ":" + str(segmin) + "-" + str(segmax),
"blueprint": blueprint,
"extra": addList
}
response = JsonResponse(retJSON)
return response
def add_study_data(self, **options):
''' add gwas stats from a study '''
study = options['study_id']
file = options['addStudyData']
message = ""
print("Deleting study hits for " + study)
Delete.docs_by_query(ElasticSettings.idx('REGION', 'STUDY_HITS'),
query=Query.term("dil_study_id", study))
with open(file, newline='') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar='|')
for row in reader:
if row[0] == 'Marker':
continue
# 0 - Marker
# 1 - disease
# 2 - Chromosome
# 3 - Region Start
# 4 - Region End
# 5 - Position
# 6 - Strand
# 7 - Major Allele
# 8 - Minor allele
# 9 - Minor allele frequency
# 10 - Discovery P value
# 11 - Discovery Odds ratio
# 12 - Discovery 95% confidence interval lower limit
# 13 - Discovery 95% confidence interval upper limit
# 14 - Replication P value
# 15 - Replication Odds ratio
# 16 - Replication 95% confidence interval lower limit
# 17 - Replication 95% confidence interval upper limit
# 18 - Combined P value
# 19 - Combined Odds ratio
# 20 - Combined 95% confidence interval lower limit
# 21 - Combined 95% confidence interval upper limit
# 22 - PP Colocalisation
# 23 - Gene
# 24 - PubMed ID
# 25 - Other Signal
# 26 - Notes
# 27 - Curation status/ failed quality control
query = ElasticQuery(Query.match("id", row[0]))
result = Search(search_query=query,
idx=ElasticSettings.idx('MARKER',
'MARKER')).search()
if result.hits_total == 0:
result2 = Search(search_query=ElasticQuery(
Query.match("rshigh", row[0])),
idx=ElasticSettings.idx(
'MARKER', 'HISTORY')).search()
if result2.hits_total > 0:
history_doc = result2.docs[0]
new_id = getattr(history_doc, "rscurrent")
query = ElasticQuery(Query.match("id", new_id))
result = Search(search_query=query,
idx=ElasticSettings.idx(
'MARKER', 'MARKER')).search()
if result.hits_total != 1:
message += "ERROR loading row of gwas data for " + row[
0] + " - Marker cannot be found; <br />\n"
marker = result.docs[0]
query = ElasticQuery(Query.match("code", row[1]))
result = Search(query,
idx=ElasticSettings.idx('DISEASE',
'DISEASE')).search()
if result.hits_total != 1:
message += "ERROR loading row of gwas data for " + row[
0] + " - Disease cannot be found; <br />\n"
continue
disease = result.docs[0]
if not re.match(r"^\w$", row[7]):
message += "ERROR loading row of gwas data for " + row[
0] + " - Major allele is not set; <br />\n"
continue
if not re.match(r"^\w$", row[8]):
message += "ERROR loading row of gwas data for " + row[
0] + " - Minor allele is not set; <br />\n"
continue
if float(row[9]) > 0.5:
message += "WARNING - MAF for " + row[
0] + " is >0.5; <br />\n"
strand = row[6]
if re.match(r"\d", strand):
strand = '+' if strand > 0 else '-'
row[6] = strand
if not re.match(r"\d+", row[2]):
row[2] = getattr(marker, "seqid")
if not re.match(r"\d+", row[5]):
row[5] = getattr(marker, "start")
if not row[5] == getattr(marker, "start"):
row[5] = getattr(marker, "start")
data = {
"chr_band": self._get_chr_band(row[2], row[5]),
"other_signal": row[25],
"species": "Human",
"disease": getattr(disease, "code"),
"notes": row[26],
"disease_locus": "TBC",
"dil_study_id": study,
"marker": getattr(marker, "id"),
"status": "N",
"pp_probability": row[22],
"tier": 100,
"pmid": row[24],
"genes": self._get_ens_gene(row[23])
}
build_info = self._get_current_build_info(row[2], row[5])
data['build_info'] = [build_info]
data['p_values'] = {
'discovery': row[10],
'replication': row[14],
'combined': row[18]
}
data['odds_ratios'] = {
'discovery': {
"or": row[11],
"lower": row[12],
"upper": row[13]
},
'replication': {
"or": row[15],
"lower": row[16],
"upper": row[17]
},
'combined': {
"or": row[19],
"lower": row[20],
"upper": row[21]
}
}
data['alleles'] = {
'major': row[7],
'minor': row[8],
'maf': row[9]
}
data['suggest'] = {'input': [], 'weight': 1}
r = Search.elastic_request(
ElasticSettings.url(),
ElasticSettings.idx('REGION', 'STUDY_HITS'),
json.dumps(data))
if r.status_code != 201:
message += "ERROR loading row of gwas data for " + row[
0] + " - Failed to create document; <br />\n"
print("\n\n" + message)
def chicpeaSearch(request, url):
queryDict = request.GET
user = request.user
targetIdx = queryDict.get("targetIdx")
blueprint = {}
hic = []
addList = []
searchType = 'gene'
searchTerm = queryDict.get("searchTerm").upper()
searchTerm = searchTerm.replace(",", "")
searchTerm = searchTerm.replace("..", "-")
snpTrack = queryDict.get("snp_track")
(idx_keys_auth, idx_type_keys_auth) = get_authenticated_idx_and_idx_types(
user=user, idx_keys=None, idx_type_keys=None)
if snpTrack:
mo = re.match(r"(.*)-(.*)", snpTrack)
(group, track) = mo.group(1, 2) # @UnusedVariable
if group != 'ud' and 'CP_STATS_'+group.upper()+'.'+snpTrack.upper() not in idx_type_keys_auth:
snpTrack = None
if targetIdx not in utils.tissues:
for target in getattr(chicp_settings, 'CP_TARGET'):
if 'CP_TARGET_'+target not in idx_keys_auth:
if targetIdx == target:
retJSON = {'error': 'Sorry, you do not have permission to view this dataset.'}
return JsonResponse(retJSON)
continue
elasticJSON = Search(idx=ElasticSettings.idx('CP_TARGET_'+target)).get_mapping(mapping_type="gene_target")
tissueList = list(elasticJSON[ElasticSettings.idx('CP_TARGET_'+target)]
['mappings']['gene_target']['_meta']['tissue_type'].keys())
utils.tissues['CP_TARGET_'+target] = tissueList
if queryDict.get("region") or re.match(r"(.*):(\d+)-(\d+)", searchTerm):
searchType = 'region'
region = searchTerm
if queryDict.get("region"):
region = queryDict.get("region")
else:
searchTerm = ""
mo = re.match(r"(.*):(\d+)-(\d+)", region)
(chrom, segmin, segmax) = mo.group(1, 2, 3)
chrom = chrom.replace('chr', "")
chrom = chrom.replace('CHR', "")
if re.search("^rs[0-9]+", searchTerm.lower()):
searchTerm = searchTerm.lower()
addList.append(_find_snp_position(snpTrack, searchTerm))
if addList[0].get("error"):
return JsonResponse({'error': addList[0]['error']})
position = addList[0]['end']
if searchType != 'region':
searchType = 'snp'
logger.warn("### "+searchType+" - "+searchTerm+' ###')
if searchType == 'region':
query_bool = BoolQuery()
filter_bool = BoolQuery()
if searchTerm and len(addList) == 0 and re.match(r"(.*):(\d+)-(\d+)",
queryDict.get("searchTerm").replace(",", "")) == None:
query_bool.must([Query.query_string(searchTerm, fields=["name", "ensg"]),
Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
else:
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
if len(addList) > 0:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
else:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", gte=segmin, lte=segmax),
RangeQuery("baitEnd", gte=segmin, lte=segmax)]),
BoolQuery(must_arr=[RangeQuery("oeStart", gte=segmin, lte=segmax),
RangeQuery("oeEnd", gte=segmin, lte=segmax)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, v1, v2) = _build_hic_query(query, targetIdx, segmin, segmax) # @UnusedVariable
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': queryDict.get("searchTerm")+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
elif searchType == 'snp':
if len(addList) > 0:
chrom = addList[0]['chr']
query_bool = BoolQuery()
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
filter_bool = BoolQuery()
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
hic, segmin, segmax = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Marker '+searchTerm+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
else:
# geneQuery = ElasticQuery.query_string(searchTerm, fields=["gene_name"])
geneQuery = ElasticQuery.filtered(Query.match_all(), Filter(Query.match("gene_name", searchTerm).query_wrap()))
resultObj = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/',
search_query=geneQuery, size=0, qsort=Sort('seqid:asc,start')).search()
if resultObj.hits_total > 1:
geneResults = []
resultObj2 = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/', search_query=geneQuery,
size=(resultObj.hits_total+1), qsort=Sort('seqid:asc,start')).search()
docs = resultObj2.docs
gene_ids = [getattr(doc, 'attr')['gene_id'][1:-1] for doc in docs]
query = ElasticQuery.filtered(Query.match_all(), TermsFilter.get_terms_filter('ensg', gene_ids))
agg = Agg('ensg_agg', "terms", {"field": "ensg", "size": 0})
res = Search(idx=ElasticSettings.idx('CP_TARGET_'+targetIdx), search_query=query, aggs=Aggs(agg),
size=0).search()
ensg_count = res.aggs['ensg_agg'].get_buckets()
gene_ids = [g['key'] for g in ensg_count]
for d in resultObj2.docs:
if getattr(d, "attr")["gene_id"].replace('\"', '') in gene_ids:
geneResults.append({
'gene_name': getattr(d, "attr")["gene_name"].replace('\"', ''),
'gene_id': getattr(d, "attr")["gene_id"].replace('\"', ''),
'location': "chr" + getattr(d, "seqid") + ":" +
locale.format_string("%d", getattr(d, "start"), grouping=True) + ".." +
locale.format_string("%d", getattr(d, "end"), grouping=True),
})
if len(geneResults) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
elif len(geneResults) > 1:
retJSON = {
'error': 'Gene name <strong>'+searchTerm+'</strong> returns too many hits, please select your prefered result from the list below.',
'results': geneResults,
'cols': ['HGNC Symbol', 'Ensembl Gene ID', 'Location']
}
return JsonResponse(retJSON)
query_bool = BoolQuery()
query_bool.must([RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
query = ElasticQuery.filtered_bool(Query.query_string(searchTerm, fields=["name", "ensg", "oeName"]),
query_bool, sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, segmin, segmax) = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
chrom = hic[0]['baitChr']
try:
chrom
except NameError:
retJSON = {'error': 'No chromosome defined for search'}
return JsonResponse(retJSON)
# get genes based on this segment
genes = _build_gene_query(chrom, segmin, segmax)
(snps, snpMeta) = _build_snp_query(snpTrack, chrom, segmin, segmax)
frags = _build_frags_query(getattr(chicp_settings, 'DEFAULT_FRAG'), chrom, segmin, segmax)
addList = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], addList)
retJSON = {"hic": hic,
"frags": frags,
"meta": {"ostart": int(segmin),
"oend": int(segmax),
"rstart": 1,
"rend": int(segmax) - int(segmin),
"rchr": str(chrom),
"tissues": utils.tissues['CP_TARGET_'+targetIdx]},
"snps": snps,
"snp_meta": snpMeta,
"genes": genes,
"region": str(chrom) + ":" + str(segmin) + "-" + str(segmax),
"blueprint": blueprint,
"extra": addList
}
response = JsonResponse(retJSON)
return response
