def _build_bigbed_query(tissue, chrom, segmin, segmax):
dataDir = os.path.join(settings.STATIC_ROOT, "chicp/data/")
bigbedData = {}
sampleLookup = getattr(chicp_settings, 'sampleLookup')
for s in sampleLookup.get(tissue):
bp = []
s_desc = ''
if s.find(":") > 0:
parts = re.split(":", s)
s = parts[0]
s_desc = parts[1]
inFile = dataDir+s+".bb"
if (os.path.exists(inFile)):
outFile = NamedTemporaryFile(delete=False)
os.system("bigBedToBed "+inFile+" "+str(outFile.name)+" -chrom=chr"+chrom+" -start="+segmin+" -end="+segmax)
with open(str(outFile.name)) as f:
for line in f:
parts = re.split(r'\t+', line.rstrip('\n'))
if len(parts) == 4:
bp.append({'start': parts[1], 'end': parts[2], 'name': parts[3], 'sample': s, 'label': s_desc,
'desc': getattr(chicp_settings, 'stateLookup').get(parts[3]).get('desc'),
'color': getattr(chicp_settings, 'stateLookup').get(parts[3]).get('color')})
else:
bp.append({'start': parts[1], 'end': parts[2], 'name': parts[3],
'color': parts[8], 'sample': s, 'label': s_desc, 'desc': parts[3]})
bp = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], bp)
bigbedData[s] = bp
return bigbedData
def _build_bigbed_query(tissue, chrom, segmin, segmax):
dataDir = os.path.join(settings.STATIC_ROOT, "chicp/data/")
bigbedData = {}
sampleLookup = getattr(chicp_settings, 'sampleLookup')
for s in sampleLookup.get(tissue):
bp = []
s_desc = ''
if s.find(":") > 0:
parts = re.split(":", s)
s = parts[0]
s_desc = parts[1]
inFile = dataDir+s+".bb"
if (os.path.exists(inFile)):
outFile = NamedTemporaryFile(delete=False)
os.system("bigBedToBed "+inFile+" "+str(outFile.name)+" -chrom=chr"+chrom+" -start="+segmin+" -end="+segmax)
with open(str(outFile.name)) as f:
for line in f:
parts = re.split(r'\t+', line.rstrip('\n'))
if len(parts) == 4:
bp.append({'start': parts[1], 'end': parts[2], 'name': parts[3], 'sample': s, 'label': s_desc,
'desc': getattr(chicp_settings, 'stateLookup').get(parts[3]).get('desc'),
'color': getattr(chicp_settings, 'stateLookup').get(parts[3]).get('color')})
else:
bp.append({'start': parts[1], 'end': parts[2], 'name': parts[3],
'color': parts[8], 'sample': s, 'label': s_desc, 'desc': parts[3]})
bp = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], bp)
bigbedData[s] = bp
return bigbedData
def _build_frags_query(frags_idx, chrom, segmin, segmax):
query = ElasticQuery.filtered(Query.terms("seqid", [chrom, str("chr"+chrom)]),
Filter(RangeQuery("end", gte=segmin, lte=segmax)),
utils.bedFields)
fragsQuery = Search(search_query=query, search_from=0, size=2000000, idx=frags_idx)
fragsResult = fragsQuery.get_result()
frags = fragsResult['data']
frags = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], frags)
return frags
def _build_gene_query(chrom, segmin, segmax):
# get genes based on this segment
geneQuery = Search.range_overlap_query(seqid=chrom, start_range=segmin, end_range=segmax, search_from=0,
size=2000, idx=getattr(chicp_settings, 'CP_GENE_IDX')+'/genes/',
field_list=utils.geneFields)
geneResult = geneQuery.get_result()
genes = geneResult['data']
genes = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], genes)
genes = [utils.flattenAttr(o) for o in genes]
for o in genes:
o.update({"bumpLevel": 0})
o.update({"length": (o['end']-o['start'])})
genes.sort(key=operator.itemgetter('length'))
return genes
def _build_frags_query(frags_idx, chrom, segmin, segmax):
query = ElasticQuery.filtered(Query.terms("seqid", [chrom, str("chr"+chrom)]),
Filter(RangeQuery("end", gte=segmin, lte=segmax)),
utils.bedFields)
fragsQuery = Search(search_query=query, search_from=0, size=10000, idx=frags_idx)
# fragsResult = fragsQuery.get_result()
# frags = fragsResult['data']
fragsResult = fragsQuery.get_json_response()
frags = []
for hit in fragsResult['hits']['hits']:
frags.append(hit['_source'])
frags = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], frags)
return frags
def _build_exon_query(chrom, segmin, segmax, genes):
# get exonic structure for genes in this section
geneExons = dict()
query_bool = BoolQuery()
query_bool.must([Query.term("seqid", chrom)])
if len(genes) > 0:
for g in genes:
query = ElasticQuery.filtered_bool(Query.query_string(g["gene_id"], fields=["name"]),
query_bool, sources=utils.snpFields)
elastic = Search(query, idx=getattr(chicp_settings, 'CP_GENE_IDX')+'/exons/', search_from=0, size=2000)
result = elastic.get_result()
exons = result['data']
exons = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], exons)
geneExons[g["gene_id"]] = sorted(exons, key=operator.itemgetter("start"))
return geneExons
def _build_hic_query(query, targetIdx, segmin=0, segmax=0):
hic = []
hicElastic = Search(query, idx=targetIdx, search_from=0, size=2000)
hicResult = hicElastic.get_result()
if len(hicResult['data']) > 0:
hic = hicResult['data']
if segmin == 0 or segmax == 0:
(segmin, segmax) = utils.segCoords(hic)
extension = int(0.05*(segmax-segmin))
segmin = segmin - extension
segmax = segmax + extension
hic = utils.makeRelative(int(segmin), int(segmax), ['baitStart', 'baitEnd', 'oeStart', 'oeEnd'], hic)
return hic, segmin, segmax
def _build_frags_query(frags_idx, chrom, segmin, segmax):
query = ElasticQuery.filtered(Query.terms("seqid", [chrom, str("chr"+chrom)]),
Filter(RangeQuery("end", gte=segmin, lte=segmax)),
utils.bedFields)
fragsQuery = Search(search_query=query, search_from=0, size=10000, idx=frags_idx)
# fragsResult = fragsQuery.get_result()
# frags = fragsResult['data']
fragsResult = fragsQuery.get_json_response()
frags = []
for hit in fragsResult['hits']['hits']:
frags.append(hit['_source'])
frags = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], frags)
return frags
def _build_hic_query(query, targetIdx, segmin=0, segmax=0):
hic = []
hicElastic = Search(query, idx=ElasticSettings.idx('CP_TARGET_'+targetIdx), search_from=0, size=2000)
# hicResult = hicElastic.get_result()
hicResult = hicElastic.get_json_response()
if "error" in hicResult:
return {'error': 'No search results found. Please try again.'}, segmin, segmax
if len(hicResult['hits']['hits']) > 0:
for hit in hicResult['hits']['hits']:
hic.append(hit['_source'])
if segmin == 0 or segmax == 0:
(segmin, segmax) = utils.segCoords(hic)
extension = int(0.05*(segmax-segmin))
segmin = segmin - extension
segmax = segmax + extension
hic = utils.makeRelative(int(segmin), int(segmax), ['baitStart', 'baitEnd', 'oeStart', 'oeEnd'], hic)
return hic, segmin, segmax
def _build_hic_query(query, targetIdx, segmin=0, segmax=0):
hic = []
hicElastic = Search(query, idx=ElasticSettings.idx('CP_TARGET_'+targetIdx), search_from=0, size=2000)
# hicResult = hicElastic.get_result()
hicResult = hicElastic.get_json_response()
if "error" in hicResult:
return {'error': 'No search results found. Please try again.'}, segmin, segmax
if len(hicResult['hits']['hits']) > 0:
for hit in hicResult['hits']['hits']:
hic.append(hit['_source'])
if segmin == 0 or segmax == 0:
(segmin, segmax) = utils.segCoords(hic)
extension = int(0.05*(segmax-segmin))
segmin = segmin - extension
segmax = segmax + extension
hic = utils.makeRelative(int(segmin), int(segmax), ['baitStart', 'baitEnd', 'oeStart', 'oeEnd'], hic)
return hic, segmin, segmax
def _build_snp_query(snp_track, chrom, segmin, segmax):
snps = []
snpMeta = {}
maxScore = -1
if snp_track and snp_track != 'None':
# get SNPs based on this segment
mo = re.match(r"(.*)-(.*)", snp_track)
(group, track) = mo.group(1, 2)
snp_track_idx = getattr(chicp_settings, 'CHICP_IDX').get(group).get('INDEX')
snp_track_type = ''
if getattr(chicp_settings, 'CHICP_IDX').get(group).get('TRACKS').get(snp_track):
snp_track_type = getattr(chicp_settings, 'CHICP_IDX').get(group).get('TRACKS') \
.get(snp_track).get('TYPE')
else:
snp_track_type = track
query = ElasticQuery.filtered(Query.terms("seqid", [chrom, str("chr"+chrom)]),
Filter(RangeQuery("end", gte=segmin, lte=segmax)),
utils.snpFields)
snpQuery = Search(search_query=query, search_from=0, size=2000000, idx=snp_track_idx+'/'+snp_track_type)
snpResult = snpQuery.get_result()
snps = snpResult['data']
snps = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], snps)
data_type = getattr(chicp_settings, 'CHICP_IDX').get(group).get('DATA_TYPE')
snpSettings = getattr(chicp_settings, 'STUDY_DEFAULTS').get(data_type)
# if 'max' in snpSettings:
# maxScore = float(snpSettings['max'])
# else:
for s in snps:
if float(s['score']) > maxScore:
maxScore = float(s['score'])
snpSettings['max'] = maxScore
snpMeta = snpSettings
return snps, snpMeta
def _build_snp_query(snp_track, chrom, segmin, segmax):
snps = []
snpMeta = {}
maxScore = -1
if snp_track and snp_track != 'None':
# get SNPs based on this segment
mo = re.match(r"(.*)-(.*)", snp_track)
(group, track) = mo.group(1, 2)
try:
snp_track_idx = ElasticSettings.idx('CP_STATS_'+group.upper(), snp_track.upper())
except SettingsError:
snp_track_idx = ElasticSettings.idx('CP_STATS_'+group.upper())+"/"+track
query = ElasticQuery.filtered(Query.terms("seqid", [chrom, str("chr"+chrom)]),
Filter(RangeQuery("end", gte=segmin, lte=segmax)),
utils.snpFields)
snpQuery = Search(search_query=query, search_from=0, size=10000, idx=snp_track_idx)
# snpResult = snpQuery.get_result()
# snps = snpResult['data']
snpResult = snpQuery.get_json_response()
snps = []
for hit in snpResult['hits']['hits']:
snps.append(hit['_source'])
snps = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], snps)
data_type = ElasticSettings.get_label('CP_STATS_'+group.upper(), None, "data_type")
snpSettings = getattr(chicp_settings, 'STUDY_DEFAULTS').get(data_type)
for s in snps:
if float(s['score']) > maxScore:
maxScore = float(s['score'])
snpSettings['max'] = maxScore
snpMeta = snpSettings
return snps, snpMeta
def _build_snp_query(snp_track, chrom, segmin, segmax):
snps = []
snpMeta = {}
maxScore = -1
if snp_track and snp_track != 'None':
# get SNPs based on this segment
mo = re.match(r"(.*)-(.*)", snp_track)
(group, track) = mo.group(1, 2)
try:
snp_track_idx = ElasticSettings.idx('CP_STATS_'+group.upper(), snp_track.upper())
except SettingsError:
snp_track_idx = ElasticSettings.idx('CP_STATS_'+group.upper())+"/"+track
query = ElasticQuery.filtered(Query.terms("seqid", [chrom, str("chr"+chrom)]),
Filter(RangeQuery("end", gte=segmin, lte=segmax)),
utils.snpFields)
snpQuery = Search(search_query=query, search_from=0, size=10000, idx=snp_track_idx)
# snpResult = snpQuery.get_result()
# snps = snpResult['data']
snpResult = snpQuery.get_json_response()
snps = []
for hit in snpResult['hits']['hits']:
snps.append(hit['_source'])
snps = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], snps)
data_type = ElasticSettings.get_label('CP_STATS_'+group.upper(), None, "data_type")
snpSettings = getattr(chicp_settings, 'STUDY_DEFAULTS').get(data_type)
for s in snps:
if float(s['score']) > maxScore:
maxScore = float(s['score'])
snpSettings['max'] = maxScore
snpMeta = snpSettings
return snps, snpMeta
def chicpeaSearch(request, url):
queryDict = request.GET
user = request.user
targetIdx = queryDict.get("targetIdx")
blueprint = {}
hic = []
addList = []
searchType = 'gene'
searchTerm = queryDict.get("searchTerm").upper()
searchTerm = searchTerm.replace(",", "")
searchTerm = searchTerm.replace("..", "-")
searchTerm = searchTerm.replace(" ", "") # Chris suggestion to prevent issue with spaces in queries
snpTrack = queryDict.get("snp_track")
(idx_keys_auth, idx_type_keys_auth) = get_authenticated_idx_and_idx_types(
user=user, idx_keys=None, idx_type_keys=None)
if snpTrack:
mo = re.match(r"(.*)-(.*)", snpTrack)
(group, track) = mo.group(1, 2) # @UnusedVariable
if group != 'ud' and 'CP_STATS_'+group.upper()+'.'+snpTrack.upper() not in idx_type_keys_auth:
snpTrack = None
if targetIdx not in utils.tissues:
for target in getattr(chicp_settings, 'CP_TARGET'):
if 'CP_TARGET_'+target not in idx_keys_auth:
if targetIdx == target:
retJSON = {'error': 'Sorry, you do not have permission to view this dataset.'}
return JsonResponse(retJSON)
continue
elasticJSON = Search(idx=ElasticSettings.idx('CP_TARGET_'+target)).get_mapping(mapping_type="gene_target")
tissueList = list(elasticJSON[ElasticSettings.idx('CP_TARGET_'+target)]
['mappings']['gene_target']['_meta']['tissue_type'].keys())
utils.tissues['CP_TARGET_'+target] = tissueList
if queryDict.get("region") or re.match(r"(.*):(\d+)-(\d+)", searchTerm):
searchType = 'region'
region = searchTerm
if queryDict.get("region"):
region = queryDict.get("region")
else:
searchTerm = ""
mo = re.match(r"(.*):(\d+)-(\d+)", region)
(chrom, segmin, segmax) = mo.group(1, 2, 3)
chrom = chrom.replace('chr', "")
chrom = chrom.replace('CHR', "")
if re.search("^rs[0-9]+", searchTerm.lower()):
searchTerm = searchTerm.lower()
addList.append(_find_snp_position(snpTrack, searchTerm))
if addList[0].get("error"):
return JsonResponse({'error': addList[0]['error']})
position = addList[0]['end']
if searchType != 'region':
searchType = 'snp'
logger.warn("### "+searchType+" - "+searchTerm+' ###')
if searchType == 'region':
query_bool = BoolQuery()
filter_bool = BoolQuery()
if searchTerm and len(addList) == 0 and re.match(r"(.*):(\d+)-(\d+)",
queryDict.get("searchTerm").replace(",", "")) == None:
query_bool.must([Query.query_string(searchTerm, fields=["name", "ensg"]),
Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
else:
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
if len(addList) > 0:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
else:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", gte=segmin, lte=segmax),
RangeQuery("baitEnd", gte=segmin, lte=segmax)]),
BoolQuery(must_arr=[RangeQuery("oeStart", gte=segmin, lte=segmax),
RangeQuery("oeEnd", gte=segmin, lte=segmax)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, v1, v2) = _build_hic_query(query, targetIdx, segmin, segmax) # @UnusedVariable
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': queryDict.get("searchTerm")+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
elif searchType == 'snp':
if len(addList) > 0:
chrom = addList[0]['chr']
query_bool = BoolQuery()
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
filter_bool = BoolQuery()
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
hic, segmin, segmax = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Marker '+searchTerm+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
else:
# geneQuery = ElasticQuery.query_string(searchTerm, fields=["gene_name"])
geneQuery = ElasticQuery.filtered(Query.match_all(), Filter(Query.match("gene_name", searchTerm).query_wrap()))
resultObj = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/',
search_query=geneQuery, size=0, qsort=Sort('seqid:asc,start')).search()
if resultObj.hits_total > 1:
geneResults = []
resultObj2 = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/', search_query=geneQuery,
size=(resultObj.hits_total+1), qsort=Sort('seqid:asc,start')).search()
docs = resultObj2.docs
gene_ids = [getattr(doc, 'attr')['gene_id'][1:-1] for doc in docs]
query = ElasticQuery.filtered(Query.match_all(), TermsFilter.get_terms_filter('ensg', gene_ids))
agg = Agg('ensg_agg', "terms", {"field": "ensg", "size": 0})
res = Search(idx=ElasticSettings.idx('CP_TARGET_'+targetIdx), search_query=query, aggs=Aggs(agg),
size=0).search()
ensg_count = res.aggs['ensg_agg'].get_buckets()
gene_ids = [g['key'] for g in ensg_count]
for d in resultObj2.docs:
if getattr(d, "attr")["gene_id"].replace('\"', '') in gene_ids:
geneResults.append({
'gene_name': getattr(d, "attr")["gene_name"].replace('\"', ''),
'gene_id': getattr(d, "attr")["gene_id"].replace('\"', ''),
'location': "chr" + getattr(d, "seqid") + ":" +
locale.format_string("%d", getattr(d, "start"), grouping=True) + ".." +
locale.format_string("%d", getattr(d, "end"), grouping=True),
})
if len(geneResults) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
elif len(geneResults) > 1:
retJSON = {
'error': 'Gene name <strong>'+searchTerm+'</strong> returns too many hits, please select your prefered result from the list below.',
'results': geneResults,
'cols': ['HGNC Symbol', 'Ensembl Gene ID', 'Location']
}
return JsonResponse(retJSON)
query_bool = BoolQuery()
query_bool.must([RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
query = ElasticQuery.filtered_bool(Query.query_string(searchTerm, fields=["name", "ensg", "oeName"]),
query_bool, sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, segmin, segmax) = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
chrom = hic[0]['baitChr']
try:
chrom
except NameError:
retJSON = {'error': 'No chromosome defined for search'}
return JsonResponse(retJSON)
# get genes based on this segment
genes = _build_gene_query(chrom, segmin, segmax)
(snps, snpMeta) = _build_snp_query(snpTrack, chrom, segmin, segmax)
frags = _build_frags_query(getattr(chicp_settings, 'DEFAULT_FRAG'), chrom, segmin, segmax)
addList = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], addList)
retJSON = {"hic": hic,
"frags": frags,
"meta": {"ostart": int(segmin),
"oend": int(segmax),
"rstart": 1,
"rend": int(segmax) - int(segmin),
"rchr": str(chrom),
"tissues": utils.tissues['CP_TARGET_'+targetIdx]},
"snps": snps,
"snp_meta": snpMeta,
"genes": genes,
"region": str(chrom) + ":" + str(segmin) + "-" + str(segmax),
"blueprint": blueprint,
"extra": addList
}
response = JsonResponse(retJSON)
return response
def chicpeaSearch(request, url):
queryDict = request.GET
targetIdx = queryDict.get("targetIdx")
blueprint = {}
hic = []
addList = []
searchType = 'gene'
searchTerm = queryDict.get("searchTerm").upper()
if targetIdx not in utils.tissues:
for idx in getattr(chicp_settings, 'TARGET_IDXS'):
elasticJSON = Search(idx=idx).get_mapping(mapping_type="gene_target")
tissueList = list(elasticJSON[idx]['mappings']['gene_target']['_meta']['tissue_type'].keys())
utils.tissues[idx] = tissueList
if queryDict.get("region") or re.match(r"(.*):(\d+)-(\d+)", queryDict.get("searchTerm")):
searchType = 'region'
region = queryDict.get("searchTerm")
if queryDict.get("region"):
region = queryDict.get("region")
else:
searchTerm = ""
mo = re.match(r"(.*):(\d+)-(\d+)", region)
(chrom, segmin, segmax) = mo.group(1, 2, 3)
chrom = chrom.replace('chr', "")
if re.search("^rs[0-9]+", queryDict.get("searchTerm").lower()):
searchTerm = queryDict.get("searchTerm").lower()
addList.append(_find_snp_position(queryDict.get("snp_track"), searchTerm))
if addList[0].get("error"):
return JsonResponse({'error': addList[0]['error']})
position = addList[0]['end']
if searchType != 'region':
searchType = 'snp'
logger.warn("### "+searchType+" - "+searchTerm+' ###')
if searchType == 'region':
query_bool = BoolQuery()
filter_bool = BoolQuery()
if searchTerm and len(addList) == 0 and re.match(r"(.*):(\d+)-(\d+)", queryDict.get("searchTerm")) == None:
query_bool.must([Query.query_string(searchTerm, fields=["name", "ensg"]),
Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
else:
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
if len(addList) > 0:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
else:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", gte=segmin, lte=segmax),
RangeQuery("baitEnd", gte=segmin, lte=segmax)]),
BoolQuery(must_arr=[RangeQuery("oeStart", gte=segmin, lte=segmax),
RangeQuery("oeEnd", gte=segmin, lte=segmax)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool, sources=utils.hicFields + utils.tissues[targetIdx])
(hic, v1, v2) = _build_hic_query(query, targetIdx, segmin, segmax)
# print(hic)
if len(hic) == 0:
retJSON = {'error': queryDict.get("searchTerm")+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
elif searchType == 'snp':
if len(addList) > 0:
chrom = addList[0]['chr']
query_bool = BoolQuery()
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
filter_bool = BoolQuery()
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues[targetIdx])
hic, segmin, segmax = _build_hic_query(query, targetIdx)
if len(hic) == 0:
retJSON = {'error': 'Marker '+searchTerm+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
else:
query_bool = BoolQuery()
query_bool.must([RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
query = ElasticQuery.filtered_bool(Query.query_string(searchTerm, fields=["name", "ensg", "oeName"]),
query_bool, sources=utils.hicFields + utils.tissues[targetIdx])
hic, segmin, segmax = _build_hic_query(query, targetIdx)
if len(hic) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
chrom = hic[0]['baitChr']
try:
chrom
except NameError:
retJSON = {'error': 'No chromosome defined for search'}
return JsonResponse(retJSON)
# get genes based on this segment
genes = _build_gene_query(chrom, segmin, segmax)
(snps, snpMeta) = _build_snp_query(queryDict.get("snp_track"), chrom, segmin, segmax)
frags = _build_frags_query(getattr(chicp_settings, 'DEFAULT_FRAG'), chrom, segmin, segmax)
addList = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], addList)
retJSON = {"hic": hic,
"frags": frags,
"meta": {"ostart": int(segmin),
"oend": int(segmax),
"rstart": 1,
"rend": int(segmax) - int(segmin),
"rchr": str(chrom),
"tissues": utils.tissues[targetIdx]},
"snps": snps,
"snp_meta": snpMeta,
"genes": genes,
"region": str(chrom) + ":" + str(segmin) + "-" + str(segmax),
"blueprint": blueprint,
"extra": addList
}
response = JsonResponse(retJSON)
return response
def chicpeaSearch(request, url):
queryDict = request.GET
user = request.user
targetIdx = queryDict.get("targetIdx")
blueprint = {}
hic = []
addList = []
searchType = 'gene'
searchTerm = queryDict.get("searchTerm").upper()
searchTerm = searchTerm.replace(",", "")
searchTerm = searchTerm.replace("..", "-")
snpTrack = queryDict.get("snp_track")
(idx_keys_auth, idx_type_keys_auth) = get_authenticated_idx_and_idx_types(
user=user, idx_keys=None, idx_type_keys=None)
if snpTrack:
mo = re.match(r"(.*)-(.*)", snpTrack)
(group, track) = mo.group(1, 2) # @UnusedVariable
if group != 'ud' and 'CP_STATS_'+group.upper()+'.'+snpTrack.upper() not in idx_type_keys_auth:
snpTrack = None
if targetIdx not in utils.tissues:
for target in getattr(chicp_settings, 'CP_TARGET'):
if 'CP_TARGET_'+target not in idx_keys_auth:
if targetIdx == target:
retJSON = {'error': 'Sorry, you do not have permission to view this dataset.'}
return JsonResponse(retJSON)
continue
elasticJSON = Search(idx=ElasticSettings.idx('CP_TARGET_'+target)).get_mapping(mapping_type="gene_target")
tissueList = list(elasticJSON[ElasticSettings.idx('CP_TARGET_'+target)]
['mappings']['gene_target']['_meta']['tissue_type'].keys())
utils.tissues['CP_TARGET_'+target] = tissueList
if queryDict.get("region") or re.match(r"(.*):(\d+)-(\d+)", searchTerm):
searchType = 'region'
region = searchTerm
if queryDict.get("region"):
region = queryDict.get("region")
else:
searchTerm = ""
mo = re.match(r"(.*):(\d+)-(\d+)", region)
(chrom, segmin, segmax) = mo.group(1, 2, 3)
chrom = chrom.replace('chr', "")
chrom = chrom.replace('CHR', "")
if re.search("^rs[0-9]+", searchTerm.lower()):
searchTerm = searchTerm.lower()
addList.append(_find_snp_position(snpTrack, searchTerm))
if addList[0].get("error"):
return JsonResponse({'error': addList[0]['error']})
position = addList[0]['end']
if searchType != 'region':
searchType = 'snp'
logger.warn("### "+searchType+" - "+searchTerm+' ###')
if searchType == 'region':
query_bool = BoolQuery()
filter_bool = BoolQuery()
if searchTerm and len(addList) == 0 and re.match(r"(.*):(\d+)-(\d+)",
queryDict.get("searchTerm").replace(",", "")) == None:
query_bool.must([Query.query_string(searchTerm, fields=["name", "ensg"]),
Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
else:
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
if len(addList) > 0:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
else:
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", gte=segmin, lte=segmax),
RangeQuery("baitEnd", gte=segmin, lte=segmax)]),
BoolQuery(must_arr=[RangeQuery("oeStart", gte=segmin, lte=segmax),
RangeQuery("oeEnd", gte=segmin, lte=segmax)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, v1, v2) = _build_hic_query(query, targetIdx, segmin, segmax) # @UnusedVariable
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': queryDict.get("searchTerm")+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
elif searchType == 'snp':
if len(addList) > 0:
chrom = addList[0]['chr']
query_bool = BoolQuery()
query_bool.must([Query.term("baitChr", chrom),
Query.term("oeChr", chrom),
RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
filter_bool = BoolQuery()
filter_bool.should([BoolQuery(must_arr=[RangeQuery("baitStart", lte=position),
RangeQuery("baitEnd", gte=position)]),
BoolQuery(must_arr=[RangeQuery("oeStart", lte=position),
RangeQuery("oeEnd", gte=position)])])
query = ElasticQuery.filtered_bool(query_bool, filter_bool,
sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
hic, segmin, segmax = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Marker '+searchTerm+' does not overlap any bait/target regions in this dataset.'}
return JsonResponse(retJSON)
else:
# geneQuery = ElasticQuery.query_string(searchTerm, fields=["gene_name"])
geneQuery = ElasticQuery.filtered(Query.match_all(), Filter(Query.match("gene_name", searchTerm).query_wrap()))
resultObj = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/',
search_query=geneQuery, size=0, qsort=Sort('seqid:asc,start')).search()
if resultObj.hits_total > 1:
geneResults = []
resultObj2 = Search(idx=getattr(chicp_settings, 'CP_GENE_IDX') + '/genes/', search_query=geneQuery,
size=(resultObj.hits_total+1), qsort=Sort('seqid:asc,start')).search()
docs = resultObj2.docs
gene_ids = [getattr(doc, 'attr')['gene_id'][1:-1] for doc in docs]
query = ElasticQuery.filtered(Query.match_all(), TermsFilter.get_terms_filter('ensg', gene_ids))
agg = Agg('ensg_agg', "terms", {"field": "ensg", "size": 0})
res = Search(idx=ElasticSettings.idx('CP_TARGET_'+targetIdx), search_query=query, aggs=Aggs(agg),
size=0).search()
ensg_count = res.aggs['ensg_agg'].get_buckets()
gene_ids = [g['key'] for g in ensg_count]
for d in resultObj2.docs:
if getattr(d, "attr")["gene_id"].replace('\"', '') in gene_ids:
geneResults.append({
'gene_name': getattr(d, "attr")["gene_name"].replace('\"', ''),
'gene_id': getattr(d, "attr")["gene_id"].replace('\"', ''),
'location': "chr" + getattr(d, "seqid") + ":" +
locale.format_string("%d", getattr(d, "start"), grouping=True) + ".." +
locale.format_string("%d", getattr(d, "end"), grouping=True),
})
if len(geneResults) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
elif len(geneResults) > 1:
retJSON = {
'error': 'Gene name <strong>'+searchTerm+'</strong> returns too many hits, please select your prefered result from the list below.',
'results': geneResults,
'cols': ['HGNC Symbol', 'Ensembl Gene ID', 'Location']
}
return JsonResponse(retJSON)
query_bool = BoolQuery()
query_bool.must([RangeQuery("dist", gte=-2e6, lte=2e6)])
query_bool = _add_tissue_filter(query_bool, targetIdx)
query = ElasticQuery.filtered_bool(Query.query_string(searchTerm, fields=["name", "ensg", "oeName"]),
query_bool, sources=utils.hicFields + utils.tissues['CP_TARGET_'+targetIdx])
(hic, segmin, segmax) = _build_hic_query(query, targetIdx)
if "error" in hic:
return JsonResponse(hic)
if len(hic) == 0:
retJSON = {'error': 'Gene name '+searchTerm+' not found in this dataset.'}
return JsonResponse(retJSON)
chrom = hic[0]['baitChr']
try:
chrom
except NameError:
retJSON = {'error': 'No chromosome defined for search'}
return JsonResponse(retJSON)
# get genes based on this segment
genes = _build_gene_query(chrom, segmin, segmax)
(snps, snpMeta) = _build_snp_query(snpTrack, chrom, segmin, segmax)
frags = _build_frags_query(getattr(chicp_settings, 'DEFAULT_FRAG'), chrom, segmin, segmax)
addList = utils.makeRelative(int(segmin), int(segmax), ['start', 'end'], addList)
retJSON = {"hic": hic,
"frags": frags,
"meta": {"ostart": int(segmin),
"oend": int(segmax),
"rstart": 1,
"rend": int(segmax) - int(segmin),
"rchr": str(chrom),
"tissues": utils.tissues['CP_TARGET_'+targetIdx]},
"snps": snps,
"snp_meta": snpMeta,
"genes": genes,
"region": str(chrom) + ":" + str(segmin) + "-" + str(segmax),
"blueprint": blueprint,
"extra": addList
}
response = JsonResponse(retJSON)
return response