def test_query(search_string): try: query = yacc.parse(search_string) except Exception as e: return str(e) return None
def __createAdvancedQuery__(query, skip, size, unique, sort=None): q = yacc.parse(query) if not unique: if sort is not None and len(sort) > 0: sortParams = list() fields = set() for (field, direction) in sort: fields.add(field) sortParams.append({field: {'order': direction}}) if field == 'dataVersion': sortParams.append({'updateVersion': {'order': 'desc', 'missing': 0, 'unmapped_type': 'long'}}) if '_score' not in fields: sortParams.append({"_score": {'order': 'desc'}}) if 'domainName' not in fields: sortParams.append({"domainName": {'order': 'asc'}}) if 'dataVersion' not in fields: sortParams.extend( [{'dataVersion': {'order': 'desc'}}, {'updateVersion': {'order': 'desc', 'missing': 0, 'unmapped_type': 'long'}}]) else: sortParams = [ {'_score': {'order': 'desc'}}, {'domainName': {'order': 'asc'}}, {'dataVersion': {'order': 'desc'}}, {'updateVersion': {'order': 'desc', 'missing': 0, 'unmapped_type': 'long'}}] q['sort'] = sortParams q['size'] = size q['from'] = skip else: q['size'] = 0 q["aggs"] = {"domains": { "terms": {"field": "domainName", "size": size, "order": {"max_score": "desc"}}, "aggs": {"max_score": {"max": {"script": "_score"}}, "top_domains": { "top_hits": { "size": 1, "sort": [{'_score': {'order': 'desc'}}, {"dataVersion": {"order": "desc"}}, {"updateVersion": {"order": "desc", "missing": 0, "unmapped_type": "long"}}]}}}}} return q
def __createAdvancedQuery__(query, skip, size, unique): q = yacc.parse(query) if not unique: q['sort'] = [ {'_score': {'order': 'desc'}}, {'domainName': {'order': 'asc'}}, {'dataVersion': {'order': 'desc'}} ] q['size'] = size q['from'] = skip else: q['size'] = 0 q["aggs"] = { "domains": { "terms":{ "field": "domainName", "size": size, "order": {"max_score": "desc"} }, "aggs": { "max_score": { "max": {"script": {"file": "_score"}} }, "top_domains":{ "top_hits":{ "size": 1, "sort": [ {'_score': {'order': 'desc'}}, {"dataVersion": {"order": "desc"}}, ] } } } } } return q
def advanced_search(search_string, skip = 0, size = 20, unique = False): #TODO XXX versions, dates, etc if not settings.ES_SCRIPTING_ENABLED: unique = False results = {'success': False} try: es = es_connector() index = '%s-*' % settings.ES_INDEX_PREFIX except ElasticsearchError as e: results['message'] = str(e) return results try: query = yacc.parse(search_string) except Exception as e: results['message'] = str(e) return results if not unique: query['sort'] = [ {'_score': {'order': 'desc'}}, {'domainName': {'order': 'asc'}}, {'dataVersion': {'order': 'desc'}} ] query['size'] = size query['from'] = skip else: query['size'] = 0 query["aggs"] = { "domains": { "terms":{ "field": "domainName", "size": size, "order": {"max_score": "desc"} }, "aggs": { "max_score": { "max": {"script": "_score"} }, "top_domains":{ "top_hits":{ "size": 1, "sort": [ {'_score': {'order': 'desc'}}, {"dataVersion": {"order": "desc"}}, ] } } } } } try: domains = es.search(index='%s-*' % settings.ES_INDEX_PREFIX, body = query, search_type='dfs_query_then_fetch') except Exception as e: results['message'] = str(e) return results if not unique: results['total'] = domains['hits']['total'] results['data'] = [] for domain in domains['hits']['hits']: pdomain = domain['_source'] # Take each key in details (if any) and stuff it in top level dict. if 'details' in pdomain: for k, v in pdomain['details'].iteritems(): pdomain[k] = v del pdomain['details'] if 'dataVersion' in pdomain: pdomain['Version'] = pdomain['dataVersion'] del pdomain['dataVersion'] results['data'].append(pdomain) results['avail'] = len(results['data']) results['skip'] = skip results['page_size'] = size results['success'] = True else: buckets = domains['aggregations']['domains']['buckets'] results['total'] = len(buckets) results['data'] = [] for bucket in buckets: domain = bucket['top_domains']['hits']['hits'][0] pdomain = domain['_source'] # Take each key in details (if any) and stuff it in top level dict. if 'details' in pdomain: for k, v in pdomain['details'].iteritems(): pdomain[k] = v del pdomain['details'] if 'dataVersion' in pdomain: pdomain['Version'] = pdomain['dataVersion'] del pdomain['dataVersion'] results['data'].append(pdomain) results['avail'] = len(buckets) results['skip'] = 0 results['page_size'] = size results['success'] = True return results
def advDataTableSearch(query, skip, pagesize, unique = False): if not settings.ES_SCRIPTING_ENABLED: unique = False results = {'success': False} results['aaData'] = [] try: es = es_connector() index = '%s-*' % settings.ES_INDEX_PREFIX except ElasticsearchError as e: results['message'] = str(e) return results try: q = yacc.parse(query) except Exception as e: results['message'] = str(e) return results if not unique: q['sort'] = [ {'_score': {'order': 'desc'}}, {'domainName': {'order': 'asc'}}, {'dataVersion': {'order': 'desc'}} ] q['size'] = pagesize q['from'] = skip else: q['size'] = 0 q["aggs"] = { "domains": { "terms":{ "field": "domainName", "size": pagesize, "order": {"max_score": "desc"} }, "aggs": { "max_score": { "max": {"script": "_score"} }, "top_domains":{ "top_hits":{ "size": 1, "sort": [ {'_score': {'order': 'desc'}}, {"dataVersion": {"order": "desc"}}, ] } } } } } import json print json.dumps(q) try: domains = es.search(index='%s-*' % settings.ES_INDEX_PREFIX, body = q, search_type = 'dfs_query_then_fetch') except Exception as e: results['message'] = str(e) return results if 'error' in domains: results['message'] = 'Error' return results if not unique: results['iTotalDisplayRecords'] = domains['hits']['total'] results['iTotalRecords'] = record_count() if domains['hits']['total'] > 0: for domain in domains['hits']['hits']: pdomain = domain['_source'] details = pdomain['details'] # Take each key in details (if any) and stuff it in top level dict. dom_arr = [" ", pdomain['domainName'], details['registrant_name'], details['contactEmail'], details['standardRegCreatedDate'], details['registrant_telephone'], pdomain['dataVersion'], "%.2f" % round(domain['_score'], 2)] results['aaData'].append(dom_arr) results['success'] = True else: buckets = domains['aggregations']['domains']['buckets'] results['iTotalDisplayRecords'] = len(buckets) results['iTotalRecords'] = len(buckets) for bucket in buckets: domain = bucket['top_domains']['hits']['hits'][0] pdomain = domain['_source'] details = pdomain['details'] # Take each key in details (if any) and stuff it in top level dict. dom_arr = [" ", pdomain['domainName'], details['registrant_name'], details['contactEmail'], details['standardRegCreatedDate'], details['registrant_telephone'], pdomain['dataVersion'], "%.2f" % round(domain['sort'][0], 2)] # For some reason the _score goes away in the aggregations if you sort by it results['aaData'].append(dom_arr) results['success'] = True return results
def __createAdvancedQuery__(query, skip, size, unique, sort=None): q = yacc.parse(query) if not unique: if sort is not None and len(sort) > 0: sortParams = list() fields = set() for (field, direction) in sort: fields.add(field) sortParams.append({field: {'order': direction}}) if field == 'dataVersion': sortParams.append({ 'updateVersion': { 'order': 'desc', 'missing': 0, 'unmapped_type': 'long' } }) if '_score' not in fields: sortParams.append({"_score": {'order': 'desc'}}) if 'domainName' not in fields: sortParams.append({"domainName": {'order': 'asc'}}) if 'dataVersion' not in fields: sortParams.extend([{ 'dataVersion': { 'order': 'desc' } }, { 'updateVersion': { 'order': 'desc', 'missing': 0, 'unmapped_type': 'long' } }]) else: sortParams = [{ '_score': { 'order': 'desc' } }, { 'domainName': { 'order': 'asc' } }, { 'dataVersion': { 'order': 'desc' } }, { 'updateVersion': { 'order': 'desc', 'missing': 0, 'unmapped_type': 'long' } }] q['sort'] = sortParams q['size'] = size q['from'] = skip else: q['size'] = 0 q["aggs"] = { "domains": { "terms": { "field": "domainName", "size": size, "order": { "max_score": "desc" } }, "aggs": { "max_score": { "max": { "script": "_score" } }, "top_domains": { "top_hits": { "size": 1, "sort": [{ '_score': { 'order': 'desc' } }, { "dataVersion": { "order": "desc" } }, { "updateVersion": { "order": "desc", "missing": 0, "unmapped_type": "long" } }] } } } } } return q
def __createAdvancedQuery__(query, skip, size, unique): q = yacc.parse(query) if not unique: q['sort'] = [{ '_score': { 'order': 'desc' } }, { 'domainName': { 'order': 'asc' } }, { 'dataVersion': { 'order': 'desc' } }, { 'updateVersion': { 'order': 'desc', 'missing': 0, 'unmapped_type': 'long' } }] q['size'] = size q['from'] = skip else: q['size'] = 0 q["aggs"] = { "domains": { "terms": { "field": "domainName", "size": size, "order": { "max_score": "desc" } }, "aggs": { "max_score": { "max": { "script": "_score" } }, "top_domains": { "top_hits": { "size": 1, "sort": [{ '_score': { 'order': 'desc' } }, { "dataVersion": { "order": "desc" } }, { "updateVersion": { "order": "desc", "missing": 0, "unmapped_type": "long" } }] } } } } } return q