def hits(self):
"""Return the docs from the response."""
raw_hits = self.raw_hits
if not raw_hits and self.query.uses_aggregations() and self.query._size == 0:
raise ESError("no hits, did you forget about no_hits_with_aggs?")
return [self.normalize_result(self.query, r) for r in raw_hits]
def _verify_is_alias(self, index_or_alias):
from corehq.elastic import ES_META, ESError
from pillowtop.tests.utils import TEST_ES_ALIAS
all_es_aliases = [index_info.alias for index_info in ES_META.values()] + [TEST_ES_ALIAS]
if index_or_alias not in all_es_aliases:
raise ESError(
f"{index_or_alias} is an unknown alias, query target must be one of {all_es_aliases}")
def run_query(self, es_query, es_type=None):
"""
Run a more advanced POST based ES query
Returns the raw query json back, or None if there's an error
"""
logger.info(
"ESlog: [%s.%s] ESquery: %s" %
(self.__class__.__name__, self.domain, json.dumps(es_query)))
if 'fields' in es_query or 'script_fields' in es_query:
#nasty hack to add domain field to query that does specific fields.
#do nothing if there's no field query because we get everything
fields = es_query.get('fields', [])
fields.append('domain')
es_query['fields'] = fields
try:
es_results = self.es_interface.search(self.index,
es_type,
body=es_query)
report_and_fail_on_shard_failures(es_results)
except ElasticsearchException as e:
if 'query_string' in es_query.get('query',
{}).get('filtered',
{}).get('query', {}):
# the error may have been caused by a bad query string
# re-run with no query string to check
querystring = es_query['query']['filtered']['query'][
'query_string']['query']
new_query = es_query
new_query['query']['filtered']['query'] = {"match_all": {}}
new_results = self.run_query(new_query)
if new_results:
# the request succeeded without that query string
# an error with a blank query will return None
raise ESUserError("Error with elasticsearch query: %s" %
querystring)
msg = "Error in elasticsearch query [%s]: %s\nquery: %s" % (
self.index, str(e), es_query)
raise ESError(msg)
hits = []
for res in es_results['hits']['hits']:
if '_source' in res:
res_domain = res['_source'].get('domain', None)
elif 'fields' in res:
res['fields'] = flatten_field_dict(res)
res_domain = res['fields'].get('domain', None)
# security check
if res_domain == self.domain:
hits.append(res)
else:
logger.info(
"Requester domain %s does not match result domain %s" %
(self.domain, res_domain))
es_results['hits']['hits'] = hits
return es_results
def run_query(self, es_query, es_type=None):
"""
Run a more advanced POST based ES query
Returns the raw query json back, or None if there's an error
"""
logging.info(
"ESlog: [%s.%s] ESquery: %s" %
(self.__class__.__name__, self.domain, simplejson.dumps(es_query)))
if 'fields' in es_query or 'script_fields' in es_query:
#nasty hack to add domain field to query that does specific fields.
#do nothing if there's no field query because we get everything
fields = es_query.get('fields', [])
fields.append('domain')
es_query['fields'] = fields
es_base = self.es[self.index] if es_type is None else self.es[
self.index][es_type]
es_results = es_base.get('_search', data=es_query)
if 'error' in es_results:
if 'query_string' in es_query.get('query',
{}).get('filtered',
{}).get('query', {}):
# the error may have been caused by a bad query string
# re-run with no query string to check
querystring = es_query['query']['filtered']['query'][
'query_string']['query']
new_query = es_query
new_query['query']['filtered']['query'] = {"match_all": {}}
new_results = self.run_query(new_query)
if new_results:
# the request succeeded without that query string
# an error with a blank query will return None
raise ESUserError("Error with elasticsearch query: %s" %
querystring)
msg = "Error in elasticsearch query [%s]: %s\nquery: %s" % (
self.index, es_results['error'], es_query)
notify_exception(None, message=msg)
raise ESError(msg)
hits = []
for res in es_results['hits']['hits']:
if '_source' in res:
res_domain = res['_source'].get('domain', None)
elif 'fields' in res:
res_domain = res['fields'].get('domain', None)
# security check
if res_domain == self.domain:
hits.append(res)
else:
logging.info(
"Requester domain %s does not match result domain %s" %
(self.domain, res_domain))
es_results['hits']['hits'] = hits
return es_results
def __init__(self, raw, query):
if 'error' in raw:
msg = ("ElasticSearch Error\n{error}\nIndex: {index}"
"\nQuery: {query}").format(
error=raw['error'],
index=query.index,
query=query.dumps(pretty=True),
)
raise ESError(msg)
self.raw = raw
self.query = query
def get_bulk_docs(self, index_alias, doc_type, doc_ids):
from corehq.elastic import ESError
self._verify_is_alias(index_alias)
docs = []
results = self.es.mget(
index=index_alias, doc_type=doc_type, body={'ids': doc_ids}, _source=True)
for doc_result in results['docs']:
if 'error' in doc_result:
raise ESError(doc_result['error'].get('reason', 'error doing bulk get'))
if doc_result['found']:
self._fix_hit(doc_result)
docs.append(doc_result['_source'])
return docs