def hits(self): """Return the docs from the response.""" raw_hits = self.raw_hits if not raw_hits and self.query.uses_aggregations() and self.query._size == 0: raise ESError("no hits, did you forget about no_hits_with_aggs?") return [self.normalize_result(self.query, r) for r in raw_hits]
def _verify_is_alias(self, index_or_alias): from corehq.elastic import ES_META, ESError from pillowtop.tests.utils import TEST_ES_ALIAS all_es_aliases = [index_info.alias for index_info in ES_META.values()] + [TEST_ES_ALIAS] if index_or_alias not in all_es_aliases: raise ESError( f"{index_or_alias} is an unknown alias, query target must be one of {all_es_aliases}")
def run_query(self, es_query, es_type=None): """ Run a more advanced POST based ES query Returns the raw query json back, or None if there's an error """ logger.info( "ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, json.dumps(es_query))) if 'fields' in es_query or 'script_fields' in es_query: #nasty hack to add domain field to query that does specific fields. #do nothing if there's no field query because we get everything fields = es_query.get('fields', []) fields.append('domain') es_query['fields'] = fields try: es_results = self.es_interface.search(self.index, es_type, body=es_query) report_and_fail_on_shard_failures(es_results) except ElasticsearchException as e: if 'query_string' in es_query.get('query', {}).get('filtered', {}).get('query', {}): # the error may have been caused by a bad query string # re-run with no query string to check querystring = es_query['query']['filtered']['query'][ 'query_string']['query'] new_query = es_query new_query['query']['filtered']['query'] = {"match_all": {}} new_results = self.run_query(new_query) if new_results: # the request succeeded without that query string # an error with a blank query will return None raise ESUserError("Error with elasticsearch query: %s" % querystring) msg = "Error in elasticsearch query [%s]: %s\nquery: %s" % ( self.index, str(e), es_query) raise ESError(msg) hits = [] for res in es_results['hits']['hits']: if '_source' in res: res_domain = res['_source'].get('domain', None) elif 'fields' in res: res['fields'] = flatten_field_dict(res) res_domain = res['fields'].get('domain', None) # security check if res_domain == self.domain: hits.append(res) else: logger.info( "Requester domain %s does not match result domain %s" % (self.domain, res_domain)) es_results['hits']['hits'] = hits return es_results
def run_query(self, es_query, es_type=None): """ Run a more advanced POST based ES query Returns the raw query json back, or None if there's an error """ logging.info( "ESlog: [%s.%s] ESquery: %s" % (self.__class__.__name__, self.domain, simplejson.dumps(es_query))) if 'fields' in es_query or 'script_fields' in es_query: #nasty hack to add domain field to query that does specific fields. #do nothing if there's no field query because we get everything fields = es_query.get('fields', []) fields.append('domain') es_query['fields'] = fields es_base = self.es[self.index] if es_type is None else self.es[ self.index][es_type] es_results = es_base.get('_search', data=es_query) if 'error' in es_results: if 'query_string' in es_query.get('query', {}).get('filtered', {}).get('query', {}): # the error may have been caused by a bad query string # re-run with no query string to check querystring = es_query['query']['filtered']['query'][ 'query_string']['query'] new_query = es_query new_query['query']['filtered']['query'] = {"match_all": {}} new_results = self.run_query(new_query) if new_results: # the request succeeded without that query string # an error with a blank query will return None raise ESUserError("Error with elasticsearch query: %s" % querystring) msg = "Error in elasticsearch query [%s]: %s\nquery: %s" % ( self.index, es_results['error'], es_query) notify_exception(None, message=msg) raise ESError(msg) hits = [] for res in es_results['hits']['hits']: if '_source' in res: res_domain = res['_source'].get('domain', None) elif 'fields' in res: res_domain = res['fields'].get('domain', None) # security check if res_domain == self.domain: hits.append(res) else: logging.info( "Requester domain %s does not match result domain %s" % (self.domain, res_domain)) es_results['hits']['hits'] = hits return es_results
def __init__(self, raw, query): if 'error' in raw: msg = ("ElasticSearch Error\n{error}\nIndex: {index}" "\nQuery: {query}").format( error=raw['error'], index=query.index, query=query.dumps(pretty=True), ) raise ESError(msg) self.raw = raw self.query = query
def get_bulk_docs(self, index_alias, doc_type, doc_ids): from corehq.elastic import ESError self._verify_is_alias(index_alias) docs = [] results = self.es.mget( index=index_alias, doc_type=doc_type, body={'ids': doc_ids}, _source=True) for doc_result in results['docs']: if 'error' in doc_result: raise ESError(doc_result['error'].get('reason', 'error doing bulk get')) if doc_result['found']: self._fix_hit(doc_result) docs.append(doc_result['_source']) return docs