def _get_distance(self): from haystack.utils.geo import Distance if self._distance is None: # We didn't get it from the backend & we haven't tried calculating # it yet. Check if geopy is available to do it the "slow" way # (even though slow meant 100 distance calculations in 0.004 seconds # in my testing). if geopy_distance is None: raise SpatialError("The backend doesn't have 'DISTANCE_AVAILABLE' enabled & the 'geopy' library could not be imported, so distance information is not available.") if not self._point_of_origin: raise SpatialError("The original point is not available.") if not hasattr(self, self._point_of_origin['field']): raise SpatialError("The field '%s' was not included in search results, so the distance could not be calculated." % self._point_of_origin['field']) po_lng, po_lat = self._point_of_origin['point'].get_coords() location_field = getattr(self, self._point_of_origin['field']) if location_field is None: return None lf_lng, lf_lat = location_field.get_coords() self._distance = Distance(km=geopy_distance.distance((po_lat, po_lng), (lf_lat, lf_lng)).km) # We've either already calculated it or the backend returned it, so # let's use that. return self._distance
def sqs_filter_l(self, sqs, search_params): location = search_params.get('l', None) if location: self.point, self.max_range = self._get_location(location) if self.point: sqs = sqs.dwithin( 'locations', self.point, Distance(km=search_params.get('r', self.max_range))) return sqs
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if hasattr(raw_results, 'stats'): stats = raw_results.stats.get('stats_fields', {}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling and hasattr(raw_results, 'spellcheck'): # Solr 5+ changed the JSON response format so the suggestions will be key-value mapped rather # than simply paired elements in a list, which is a nice improvement but incompatible with # Solr 4: https://issues.apache.org/jira/browse/SOLR-3029 if len(raw_results.spellcheck.get('collations', [])): spelling_suggestion = raw_results.spellcheck['collations'][-1] elif len(raw_results.spellcheck.get('suggestions', [])): spelling_suggestion = raw_results.spellcheck['suggestions'][-1] assert spelling_suggestion is None or isinstance( spelling_suggestion, six.string_types) unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python( value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) del (additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields[ 'highlighted'] = raw_results.highlighting[ raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if hasattr(raw_results,'stats'): stats = raw_results.stats.get('stats_fields',{}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list(zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling is True: if hasattr(raw_results, 'spellcheck'): if len(raw_results.spellcheck.get('suggestions', [])): # For some reason, it's an array of pairs. Pull off the # collated result from the end. spelling_suggestion = raw_results.spellcheck.get('suggestions')[-1] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = apps.get_model(app_label, model_name) if model and model in indexed_models: for key, value in raw_result.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = custom_to_python(value, self.conn._to_python) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) del(additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = spelling_suggestions = None if result_class is None: result_class = SearchResult if hasattr(raw_results, "stats"): stats = raw_results.stats.get("stats_fields", {}) if hasattr(raw_results, "facets"): facets = { "fields": raw_results.facets.get("facet_fields", {}), "dates": raw_results.facets.get("facet_dates", {}), "queries": raw_results.facets.get("facet_queries", {}), } for key in ["fields"]: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip( facets[key][facet_field][::2], facets[key][facet_field][1::2], )) if self.include_spelling and hasattr(raw_results, "spellcheck"): try: spelling_suggestions = self.extract_spelling_suggestions( raw_results) except Exception as exc: self.log.error( "Error extracting spelling suggestions: %s", exc, exc_info=True, extra={"data": { "spellcheck": raw_results.spellcheck }}, ) if not self.silently_fail: raise spelling_suggestions = None if spelling_suggestions: # Maintain compatibility with older versions of Haystack which returned a single suggestion: spelling_suggestion = spelling_suggestions[-1] assert isinstance(spelling_suggestion, six.string_types) else: spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], "convert"): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python( value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) del (additional_fields["score"]) if raw_result[ID] in getattr(raw_results, "highlighting", {}): additional_fields[ "highlighted"] = raw_results.highlighting[ raw_result[ID]] if distance_point: additional_fields["_point_of_origin"] = distance_point if raw_result.get("__dist__"): from haystack.utils.geo import Distance additional_fields["_distance"] = Distance( km=float(raw_result["__dist__"])) else: additional_fields["_distance"] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result["score"], **additional_fields) results.append(result) else: hits -= 1 return { "results": results, "hits": hits, "stats": stats, "facets": facets, "spelling_suggestion": spelling_suggestion, "spelling_suggestions": spelling_suggestions, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest']['suggest'] spelling_suggestion = ' '.join([word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest]) if 'facets' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } for facet_fieldname, facet_info in raw_results['facets'].items(): if facet_info.get('_type', 'terms') == 'terms': facets['fields'][facet_fieldname] = [(individual['term'], individual['count']) for individual in facet_info['terms']] elif facet_info.get('_type', 'terms') == 'date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates'][facet_fieldname] = [(datetime.datetime.utcfromtimestamp(individual['time'] / 1000), individual['count']) for individual in facet_info['entries']] elif facet_info.get('_type', 'terms') == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get('hits', {}).get('hits', []): source = raw_result['_source'] app_label, model_name = source[DJANGO_CT].split('.') additional_fields = {} model = get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result['highlight'].get(content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=-1): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest'].get('suggest') if raw_suggest: spelling_suggestion = ' '.join([ word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest ]) if 'aggregations' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } # ES can return negative timestamps for pre-1970 data. Handle it. def from_timestamp(tm): if tm >= 0: return datetime.utcfromtimestamp(tm) else: return datetime(1970, 1, 1) + timedelta(seconds=tm) for facet_fieldname, facet_info in raw_results[ 'aggregations'].items(): try: facet_type = facet_info['meta']['_type'] except KeyError: facet_type = 'terms' if facet_type == 'terms': facets['fields'][facet_fieldname] = [ (bucket['key'], bucket['doc_count']) for bucket in facet_info.get('buckets', []) ] elif facet_type == 'haystack_date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. dates = [(from_timestamp(bucket['key'] / 1000), bucket['doc_count']) for bucket in facet_info.get('buckets', [])] facets['dates'][facet_fieldname[:-len( DATE_HISTOGRAM_FIELD_NAME_SUFFIX)]] = dates elif facet_type == 'haystack_date_range': pass elif facet_type == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.get('hits', {}).get('hits', []): source = raw_result['_source'] app_label, model_name = source[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result['highlight'] if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort >= 0 and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['sort'][geo_sort])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results['hits'].total facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if 'facets' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } # ES can return negative timestamps for pre-1970 data. Handle it. def from_timestamp(tm): if tm >= 0: return datetime.utcfromtimestamp(tm) else: return datetime(1970, 1, 1) + timedelta(seconds=tm) for facet_fieldname, facet_info in raw_results['facets'].items(): if facet_info.get('_type', 'terms') == 'terms': facets['fields'][facet_fieldname] = [ (individual['term'], individual['count']) for individual in facet_info['terms'] ] elif facet_info.get('_type', 'terms') == 'date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates'][facet_fieldname] = [ (from_timestamp(individual['time'] / 1000), individual['count']) for individual in facet_info['entries'] ] elif facet_info.get('_type', 'terms') == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() content_field = unified_index.document_field # articleids = list(map(lambda x: x['_id'], raw_results['hits']['hits'])) # article_results = list(Article.objects.filter(id__in=articleids)) for raw_result in raw_results['hits']['hits']: app_label = 'blog' model_name = 'Article' additional_fields = {} if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result['highlight'].get( content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result['_id'], raw_result['_score'], **additional_fields) results.append(result) return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest'].get('suggest') if raw_suggest: spelling_suggestion = ' '.join([word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest]) if FACET_RESULTS in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } # ES can return negative timestamps for pre-1970 data. Handle it. def from_timestamp(tm): if elasticsearch.__version__[0] < 2: if tm >= 0: return datetime.utcfromtimestamp(tm) else: return datetime(1970, 1, 1) + timedelta(seconds=tm) else: return datetime.strptime(tm, '%Y-%m-%dT%H:%M:%S.%fZ') if elasticsearch.__version__[0] < 2: for facet_fieldname, facet_info in raw_results['facets'].items(): if facet_info.get('_type', 'terms') == 'terms': facets['fields'][facet_fieldname] = [(individual['term'], individual['count']) for individual in facet_info['terms']] elif facet_info.get('_type', 'terms') == 'date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates'][facet_fieldname] = [(from_timestamp(individual['time'] / 1000), individual['count']) for individual in facet_info['entries']] elif facet_info.get('_type', 'terms') == 'query': facets['queries'][facet_fieldname] = facet_info['count'] else: # In version 2 Elasticsearch moved facets into a subset of aggs # as well as getting rid of _type in facets so we have to do a # bit more guess work to figure out what is inside. # They also nested aggs inside another level which we call filter1. # But only sometimes. for facet_fieldname, facet_info in (raw_results[FACET_RESULTS].get('filter1') or raw_results[FACET_RESULTS]).items(): if 'pub_date' in facet_fieldname: # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates']['pub_date'] = [(from_timestamp( (individual['key_as_string'] / 1000) if elasticsearch.__version__[0] < 2 else individual['key_as_string']), individual['doc_count']) for individual in facet_info['buckets']] elif 'buckets' == facet_fieldname: facets['fields'][facet_fieldname] = [(individual['key'], individual['doc_count']) for individual in facet_info['buckets']] elif isinstance(facet_info, dict) and 'buckets' in facet_info: facets['fields'][facet_fieldname] = [(individual['key'], individual['doc_count']) for individual in facet_info['buckets']] elif isinstance(facet_info, dict) and 'doc_count' in facet_info: facets['queries'][facet_fieldname] = facet_info['doc_count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get('hits', {}).get('hits', []): source = raw_result['_source'] app_label, model_name = source[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result['highlight'].get(content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results( self, raw_results, model=None, highlight=False, result_class=None, distance_point=None, percent_score=False, is_faceted=False, ): results = [] if len(raw_results) == 1 and "rows_count" in raw_results[0]: hits = raw_results[0]["rows_count"] else: # we can't rely on the DSE response for the hits if we're not using the COUNT(*), that's why we set as 0 and # if we need the hits later we will do a COUNT(*) query. hits = 0 facets = {} stats = {} spelling_suggestion = spelling_suggestions = None if is_faceted: assert len(raw_results ) == 1, "Faceted searches should have only one result" raw_results = raw_results[0] facets["fields"] = json.loads(raw_results.get( "facet_fields", "{}")) facets["dates"] = json.loads(raw_results.get("facet_dates", "{}")) facets["queries"] = json.loads( raw_results.get("facet_queries", "{}")) if "facet_heatmaps" in raw_results: facets["heatmaps"] = json.loads( raw_results.get("facet_heatmaps", "{}")) if result_class is None: result_class = SearchResult if hasattr(raw_results, "stats"): stats = raw_results.stats.get("stats_fields", {}) if "facet_fields" in raw_results: for key in ["fields"]: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip(facets[key][facet_field], facets[key][facet_field].values())) from haystack import connections unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() if not is_faceted: for raw_result in raw_results: app_label, model_name = raw_result[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if isinstance(value, Date): value = str(value) if string_key in index.fields and hasattr( index.fields[string_key], "convert"): additional_fields[string_key] = index.fields[ string_key].convert(value) elif hasattr(model, string_key): column = getattr(model, string_key) if hasattr(column, "column"): additional_fields[ string_key] = column.column.to_python( value) elif hasattr(column, "to_python"): additional_fields[ string_key] = column.to_python(value) else: additional_fields[ string_key] = self.conn._to_python(value) else: additional_fields[ string_key] = self.conn._to_python(value) del additional_fields[DJANGO_CT] del additional_fields[DJANGO_ID] if "rows_count" in additional_fields: del additional_fields["rows_count"] if raw_result[ID] in getattr(raw_results, "highlighting", {}): additional_fields[ "highlighted"] = raw_results.highlighting[ raw_result[ID]] if distance_point: additional_fields["_point_of_origin"] = distance_point if raw_result.get("__dist__"): from haystack.utils.geo import Distance additional_fields["_distance"] = Distance( km=float(raw_result["__dist__"])) else: additional_fields["_distance"] = None additional_fields["already_loaded"] = True if "score" not in additional_fields: additional_fields["score"] = 1.0 result = result_class(app_label, model_name, raw_result[DJANGO_ID], **additional_fields) results.append(result) results = { "results": results, "hits": hits, "stats": stats, "facets": facets, "spelling_suggestion": spelling_suggestion, "spelling_suggestions": spelling_suggestions, } if "facet_ranges" in raw_results: results["facets"]["ranges"] = {} ranges = json.loads(raw_results.get("facet_ranges", "{}")) if len(ranges): for field_name, range_data in ranges.items(): if field_name in self.date_facets: results["facets"]["dates"][field_name] = tuple( zip(range_data["counts"], range_data["counts"].values())) elif field_name in self.range_facets: results["facets"]["ranges"][field_name] = tuple( zip(range_data["counts"], range_data["counts"].values())) return results
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest'].get('suggest') if raw_suggest: spelling_suggestion = ' '.join([ word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest ]) if 'facets' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } for facet_fieldname, facet_info in raw_results['facets'].items(): if facet_info.get('_type', 'terms') == 'terms': facets['fields'][facet_fieldname] = [ (individual['term'], individual['count']) for individual in facet_info['terms'] ] elif facet_info.get('_type', 'terms') == 'date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates'][facet_fieldname] = [ (datetime.datetime.utcfromtimestamp( individual['time'] / 1000), individual['count']) for individual in facet_info['entries'] ] elif facet_info.get('_type', 'terms') == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get('hits', {}).get('hits', []): result_type = raw_result['_type'] app_model_delim_idx = result_type.rfind('_') app_label = result_type[:app_model_delim_idx] model_name = result_type[app_model_delim_idx + 1:] model = get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) # Look for _source, then _fields in case this was a values_list query if '_source' in raw_result: source = raw_result['_source'] elif 'fields' in raw_result: # _source wasn't requested, but we have fields, so this was a # values_list query. All Elasticsearch's fields results are lists # so we'll need to fix that for anything that isn't multivalued raw_fields = raw_result['fields'] for field in raw_fields: # This breaks multi-value fields when they have one value, so gotta # sort out a fix if field in index.fields and index.fields[ field].is_multivalued: # leave this as a list continue else: raw_fields[field] = raw_fields[field][0] source = raw_fields additional_fields = {} for key, value in source.items(): string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) if DJANGO_CT in additional_fields: del (additional_fields[DJANGO_CT]) if DJANGO_ID in additional_fields: del (additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result[ 'highlight'].get(content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results( self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False, ): from haystack import connections results = [] hits = raw_results.get("hits", {}).get("total", 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and "suggest" in raw_results: raw_suggest = raw_results["suggest"].get("suggest") if raw_suggest: spelling_suggestion = " ".join([ word["text"] if len(word["options"]) == 0 else word["options"][0]["text"] for word in raw_suggest ]) if "facets" in raw_results: facets = {"fields": {}, "dates": {}, "queries": {}} # ES can return negative timestamps for pre-1970 data. Handle it. def from_timestamp(tm): if tm >= 0: return datetime.utcfromtimestamp(tm) else: return datetime(1970, 1, 1) + timedelta(seconds=tm) for facet_fieldname, facet_info in raw_results["facets"].items(): if facet_info.get("_type", "terms") == "terms": facets["fields"][facet_fieldname] = [ (individual["term"], individual["count"]) for individual in facet_info["terms"] ] elif facet_info.get("_type", "terms") == "date_histogram": # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets["dates"][facet_fieldname] = [ (from_timestamp(individual["time"] / 1000), individual["count"]) for individual in facet_info["entries"] ] elif facet_info.get("_type", "terms") == "query": facets["queries"][facet_fieldname] = facet_info["count"] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get("hits", {}).get("hits", []): source = raw_result["_source"] app_label, model_name = source[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = source and unified_index.get_index(model) for key, value in source.items(): string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], "convert"): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if "highlight" in raw_result: additional_fields["highlighted"] = raw_result[ "highlight"].get(content_field, "") if distance_point: additional_fields["_point_of_origin"] = distance_point if geo_sort and raw_result.get("sort"): from haystack.utils.geo import Distance additional_fields["_distance"] = Distance( km=float(raw_result["sort"][0])) else: additional_fields["_distance"] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result["_score"], **additional_fields) results.append(result) else: hits -= 1 return { "results": results, "hits": hits, "facets": facets, "spelling_suggestion": spelling_suggestion, }