def test_query(search_string):
try:
query = yacc.parse(search_string)
except Exception as e:
return str(e)
return None
def test_query(search_string):
try:
query = yacc.parse(search_string)
except Exception as e:
return str(e)
return None
def __createAdvancedQuery__(query, skip, size, unique, sort=None):
q = yacc.parse(query)
if not unique:
if sort is not None and len(sort) > 0:
sortParams = list()
fields = set()
for (field, direction) in sort:
fields.add(field)
sortParams.append({field: {'order': direction}})
if field == 'dataVersion':
sortParams.append({'updateVersion':
{'order': 'desc',
'missing': 0,
'unmapped_type': 'long'}})
if '_score' not in fields:
sortParams.append({"_score": {'order': 'desc'}})
if 'domainName' not in fields:
sortParams.append({"domainName": {'order': 'asc'}})
if 'dataVersion' not in fields:
sortParams.extend(
[{'dataVersion': {'order': 'desc'}},
{'updateVersion': {'order': 'desc',
'missing': 0,
'unmapped_type': 'long'}}])
else:
sortParams = [
{'_score': {'order': 'desc'}},
{'domainName': {'order': 'asc'}},
{'dataVersion': {'order': 'desc'}},
{'updateVersion': {'order': 'desc',
'missing': 0,
'unmapped_type': 'long'}}]
q['sort'] = sortParams
q['size'] = size
q['from'] = skip
else:
q['size'] = 0
q["aggs"] = {"domains": {
"terms": {"field": "domainName",
"size": size,
"order": {"max_score": "desc"}},
"aggs": {"max_score": {"max": {"script": "_score"}},
"top_domains": {
"top_hits": {
"size": 1,
"sort":
[{'_score': {'order': 'desc'}},
{"dataVersion": {"order": "desc"}},
{"updateVersion":
{"order": "desc",
"missing": 0,
"unmapped_type": "long"}}]}}}}}
return q
def __createAdvancedQuery__(query, skip, size, unique):
q = yacc.parse(query)
if not unique:
q['sort'] = [
{'_score': {'order': 'desc'}},
{'domainName': {'order': 'asc'}},
{'dataVersion': {'order': 'desc'}}
]
q['size'] = size
q['from'] = skip
else:
q['size'] = 0
q["aggs"] = {
"domains": {
"terms":{
"field": "domainName",
"size": size,
"order": {"max_score": "desc"}
},
"aggs": {
"max_score": {
"max": {"script": {"file": "_score"}}
},
"top_domains":{
"top_hits":{
"size": 1,
"sort": [
{'_score': {'order': 'desc'}},
{"dataVersion": {"order": "desc"}},
]
}
}
}
}
}
return q
def advanced_search(search_string, skip = 0, size = 20, unique = False): #TODO XXX versions, dates, etc
if not settings.ES_SCRIPTING_ENABLED:
unique = False
results = {'success': False}
try:
es = es_connector()
index = '%s-*' % settings.ES_INDEX_PREFIX
except ElasticsearchError as e:
results['message'] = str(e)
return results
try:
query = yacc.parse(search_string)
except Exception as e:
results['message'] = str(e)
return results
if not unique:
query['sort'] = [
{'_score': {'order': 'desc'}},
{'domainName': {'order': 'asc'}},
{'dataVersion': {'order': 'desc'}}
]
query['size'] = size
query['from'] = skip
else:
query['size'] = 0
query["aggs"] = {
"domains": {
"terms":{
"field": "domainName",
"size": size,
"order": {"max_score": "desc"}
},
"aggs": {
"max_score": {
"max": {"script": "_score"}
},
"top_domains":{
"top_hits":{
"size": 1,
"sort": [
{'_score': {'order': 'desc'}},
{"dataVersion": {"order": "desc"}},
]
}
}
}
}
}
try:
domains = es.search(index='%s-*' % settings.ES_INDEX_PREFIX, body = query, search_type='dfs_query_then_fetch')
except Exception as e:
results['message'] = str(e)
return results
if not unique:
results['total'] = domains['hits']['total']
results['data'] = []
for domain in domains['hits']['hits']:
pdomain = domain['_source']
# Take each key in details (if any) and stuff it in top level dict.
if 'details' in pdomain:
for k, v in pdomain['details'].iteritems():
pdomain[k] = v
del pdomain['details']
if 'dataVersion' in pdomain:
pdomain['Version'] = pdomain['dataVersion']
del pdomain['dataVersion']
results['data'].append(pdomain)
results['avail'] = len(results['data'])
results['skip'] = skip
results['page_size'] = size
results['success'] = True
else:
buckets = domains['aggregations']['domains']['buckets']
results['total'] = len(buckets)
results['data'] = []
for bucket in buckets:
domain = bucket['top_domains']['hits']['hits'][0]
pdomain = domain['_source']
# Take each key in details (if any) and stuff it in top level dict.
if 'details' in pdomain:
for k, v in pdomain['details'].iteritems():
pdomain[k] = v
del pdomain['details']
if 'dataVersion' in pdomain:
pdomain['Version'] = pdomain['dataVersion']
del pdomain['dataVersion']
results['data'].append(pdomain)
results['avail'] = len(buckets)
results['skip'] = 0
results['page_size'] = size
results['success'] = True
return results
def advDataTableSearch(query, skip, pagesize, unique = False):
if not settings.ES_SCRIPTING_ENABLED:
unique = False
results = {'success': False}
results['aaData'] = []
try:
es = es_connector()
index = '%s-*' % settings.ES_INDEX_PREFIX
except ElasticsearchError as e:
results['message'] = str(e)
return results
try:
q = yacc.parse(query)
except Exception as e:
results['message'] = str(e)
return results
if not unique:
q['sort'] = [
{'_score': {'order': 'desc'}},
{'domainName': {'order': 'asc'}},
{'dataVersion': {'order': 'desc'}}
]
q['size'] = pagesize
q['from'] = skip
else:
q['size'] = 0
q["aggs"] = {
"domains": {
"terms":{
"field": "domainName",
"size": pagesize,
"order": {"max_score": "desc"}
},
"aggs": {
"max_score": {
"max": {"script": "_score"}
},
"top_domains":{
"top_hits":{
"size": 1,
"sort": [
{'_score': {'order': 'desc'}},
{"dataVersion": {"order": "desc"}},
]
}
}
}
}
}
import json
print json.dumps(q)
try:
domains = es.search(index='%s-*' % settings.ES_INDEX_PREFIX, body = q, search_type = 'dfs_query_then_fetch')
except Exception as e:
results['message'] = str(e)
return results
if 'error' in domains:
results['message'] = 'Error'
return results
if not unique:
results['iTotalDisplayRecords'] = domains['hits']['total']
results['iTotalRecords'] = record_count()
if domains['hits']['total'] > 0:
for domain in domains['hits']['hits']:
pdomain = domain['_source']
details = pdomain['details']
# Take each key in details (if any) and stuff it in top level dict.
dom_arr = [" ", pdomain['domainName'],
details['registrant_name'], details['contactEmail'],
details['standardRegCreatedDate'], details['registrant_telephone'],
pdomain['dataVersion'], "%.2f" % round(domain['_score'], 2)]
results['aaData'].append(dom_arr)
results['success'] = True
else:
buckets = domains['aggregations']['domains']['buckets']
results['iTotalDisplayRecords'] = len(buckets)
results['iTotalRecords'] = len(buckets)
for bucket in buckets:
domain = bucket['top_domains']['hits']['hits'][0]
pdomain = domain['_source']
details = pdomain['details']
# Take each key in details (if any) and stuff it in top level dict.
dom_arr = [" ", pdomain['domainName'],
details['registrant_name'], details['contactEmail'],
details['standardRegCreatedDate'], details['registrant_telephone'],
pdomain['dataVersion'], "%.2f" % round(domain['sort'][0], 2)] # For some reason the _score goes away in the aggregations if you sort by it
results['aaData'].append(dom_arr)
results['success'] = True
return results
def __createAdvancedQuery__(query, skip, size, unique, sort=None):
q = yacc.parse(query)
if not unique:
if sort is not None and len(sort) > 0:
sortParams = list()
fields = set()
for (field, direction) in sort:
fields.add(field)
sortParams.append({field: {'order': direction}})
if field == 'dataVersion':
sortParams.append({
'updateVersion': {
'order': 'desc',
'missing': 0,
'unmapped_type': 'long'
}
})
if '_score' not in fields:
sortParams.append({"_score": {'order': 'desc'}})
if 'domainName' not in fields:
sortParams.append({"domainName": {'order': 'asc'}})
if 'dataVersion' not in fields:
sortParams.extend([{
'dataVersion': {
'order': 'desc'
}
}, {
'updateVersion': {
'order': 'desc',
'missing': 0,
'unmapped_type': 'long'
}
}])
else:
sortParams = [{
'_score': {
'order': 'desc'
}
}, {
'domainName': {
'order': 'asc'
}
}, {
'dataVersion': {
'order': 'desc'
}
}, {
'updateVersion': {
'order': 'desc',
'missing': 0,
'unmapped_type': 'long'
}
}]
q['sort'] = sortParams
q['size'] = size
q['from'] = skip
else:
q['size'] = 0
q["aggs"] = {
"domains": {
"terms": {
"field": "domainName",
"size": size,
"order": {
"max_score": "desc"
}
},
"aggs": {
"max_score": {
"max": {
"script": "_score"
}
},
"top_domains": {
"top_hits": {
"size":
1,
"sort": [{
'_score': {
'order': 'desc'
}
}, {
"dataVersion": {
"order": "desc"
}
}, {
"updateVersion": {
"order": "desc",
"missing": 0,
"unmapped_type": "long"
}
}]
}
}
}
}
}
return q
def __createAdvancedQuery__(query, skip, size, unique):
q = yacc.parse(query)
if not unique:
q['sort'] = [{
'_score': {
'order': 'desc'
}
}, {
'domainName': {
'order': 'asc'
}
}, {
'dataVersion': {
'order': 'desc'
}
}, {
'updateVersion': {
'order': 'desc',
'missing': 0,
'unmapped_type': 'long'
}
}]
q['size'] = size
q['from'] = skip
else:
q['size'] = 0
q["aggs"] = {
"domains": {
"terms": {
"field": "domainName",
"size": size,
"order": {
"max_score": "desc"
}
},
"aggs": {
"max_score": {
"max": {
"script": "_score"
}
},
"top_domains": {
"top_hits": {
"size":
1,
"sort": [{
'_score': {
'order': 'desc'
}
}, {
"dataVersion": {
"order": "desc"
}
}, {
"updateVersion": {
"order": "desc",
"missing": 0,
"unmapped_type": "long"
}
}]
}
}
}
}
}
return q
