def render(self, context): try: model_instance = self.model.resolve(context) sqs = SearchQuerySet() if self.for_types is not None: intermediate = template.Variable(self.for_types) for_types = intermediate.resolve(context).split(",") search_models = [] for model in for_types: model_class = haystack_get_model(*model.split(".")) if model_class: search_models.append(model_class) sqs = sqs.models(*search_models) sqs = sqs.more_like_this(model_instance) if self.limit is not None: sqs = sqs[: self.limit] context[self.varname] = sqs except Exception as exc: logging.warning( "Unhandled exception rendering %r: %s", self, exc, exc_info=True ) return ""
def process_documents(self, doclist, raw_results): # TODO: tame import spaghetti from haystack import connections engine = connections["default"] conn = engine.get_backend().conn unified_index = engine.get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in doclist: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in raw_result.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = conn._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) del(additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]] result = SearchResult(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) yield result
def render(self, context): try: model_instance = self.model.resolve(context) sqs = SearchQuerySet() if not self.for_types is None: intermediate = template.Variable(self.for_types) for_types = intermediate.resolve(context).split(',') search_models = [] for model in for_types: model_class = haystack_get_model(*model.split('.')) if model_class: search_models.append(model_class) sqs = sqs.models(*search_models) sqs = sqs.more_like_this(model_instance) if not self.limit is None: sqs = sqs[:self.limit] context[self.varname] = sqs except: pass return ''
def get_models(self): """Return an alphabetical list of model classes in the index.""" search_models = [] if self.is_valid(): for model in self.cleaned_data['models']: search_models.append(haystack_get_model(*model.split('.'))) return search_models
def get_models(self): """Return a list of the selected models.""" search_models = [] if self.is_valid(): for model in self.cleaned_data["models"]: search_models.append(haystack_get_model(*model.split("."))) return search_models
def _get_model(self): if self._model is None: try: self._model = haystack_get_model(self.app_label, self.model_name) except LookupError: # this changed in change 1.7 to throw an error instead of # returning None when the model isn't found. So catch the # lookup error and keep self._model == None. pass return self._model
def process_documents(self, doclist): # TODO: tame import spaghetti from haystack import connections engine = connections["default"] unified_index = engine.get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in doclist: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in raw_result.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) result = SearchResult(app_label, model_name, raw_result[DJANGO_ID], 1, **additional_fields) yield result
def _process_results( self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False, ): from haystack import connections results = [] hits = raw_results.get("hits", {}).get("total", 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and "suggest" in raw_results: raw_suggest = raw_results["suggest"].get("suggest") if raw_suggest: spelling_suggestion = " ".join([ word["text"] if len(word["options"]) == 0 else word["options"][0]["text"] for word in raw_suggest ]) if "facets" in raw_results: facets = {"fields": {}, "dates": {}, "queries": {}} # ES can return negative timestamps for pre-1970 data. Handle it. def from_timestamp(tm): if tm >= 0: return datetime.utcfromtimestamp(tm) else: return datetime(1970, 1, 1) + timedelta(seconds=tm) for facet_fieldname, facet_info in raw_results["facets"].items(): if facet_info.get("_type", "terms") == "terms": facets["fields"][facet_fieldname] = [ (individual["term"], individual["count"]) for individual in facet_info["terms"] ] elif facet_info.get("_type", "terms") == "date_histogram": # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets["dates"][facet_fieldname] = [ (from_timestamp(individual["time"] / 1000), individual["count"]) for individual in facet_info["entries"] ] elif facet_info.get("_type", "terms") == "query": facets["queries"][facet_fieldname] = facet_info["count"] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get("hits", {}).get("hits", []): source = raw_result["_source"] app_label, model_name = source[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = source and unified_index.get_index(model) for key, value in source.items(): string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], "convert"): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if "highlight" in raw_result: additional_fields["highlighted"] = raw_result[ "highlight"].get(content_field, "") if distance_point: additional_fields["_point_of_origin"] = distance_point if geo_sort and raw_result.get("sort"): from haystack.utils.geo import Distance additional_fields["_distance"] = Distance( km=float(raw_result["sort"][0])) else: additional_fields["_distance"] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result["_score"], **additional_fields) results.append(result) else: hits -= 1 return { "results": results, "hits": hits, "facets": facets, "spelling_suggestion": spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = spelling_suggestions = None if result_class is None: result_class = SearchResult if hasattr(raw_results, 'stats'): stats = raw_results.stats.get('stats_fields', {}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list(zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling and hasattr(raw_results, 'spellcheck'): try: spelling_suggestions = self.extract_spelling_suggestions(raw_results) except Exception as exc: self.log.error('Error extracting spelling suggestions: %s', exc, exc_info=True, extra={'data': {'spellcheck': raw_results.spellcheck}}) if not self.silently_fail: raise spelling_suggestions = None if spelling_suggestions: # Maintain compatibility with older versions of Haystack which returned a single suggestion: spelling_suggestion = spelling_suggestions[-1] assert isinstance(spelling_suggestion, six.string_types) else: spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) del(additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, 'spelling_suggestions': spelling_suggestions, }
def _process_results( self, raw_results, highlight=False, result_class=None, distance_point=None ): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = spelling_suggestions = None if result_class is None: result_class = SearchResult if hasattr(raw_results, "stats"): stats = raw_results.stats.get("stats_fields", {}) if hasattr(raw_results, "facets"): facets = { "fields": raw_results.facets.get("facet_fields", {}), "dates": raw_results.facets.get("facet_dates", {}), "queries": raw_results.facets.get("facet_queries", {}), "ranges": raw_results.facets.get("facet_ranges", {}), } for key in ["fields"]: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip( facets[key][facet_field][::2], facets[key][facet_field][1::2], ) ) for key in ["ranges"]: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip( facets[key][facet_field]["counts"][::2], facets[key][facet_field]["counts"][1::2], ) ) if self.include_spelling and hasattr(raw_results, "spellcheck"): try: spelling_suggestions = self.extract_spelling_suggestions(raw_results) except Exception as exc: self.log.error( "Error extracting spelling suggestions: %s", exc, exc_info=True, extra={"data": {"spellcheck": raw_results.spellcheck}}, ) if not self.silently_fail: raise spelling_suggestions = None if spelling_suggestions: # Maintain compatibility with older versions of Haystack which returned a single suggestion: spelling_suggestion = spelling_suggestions[-1] assert isinstance(spelling_suggestion, str) else: spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], "convert" ): additional_fields[string_key] = index.fields[ string_key ].convert(value) else: additional_fields[string_key] = self.conn._to_python(value) del additional_fields[DJANGO_CT] del additional_fields[DJANGO_ID] del additional_fields["score"] if raw_result[ID] in getattr(raw_results, "highlighting", {}): additional_fields["highlighted"] = raw_results.highlighting[ raw_result[ID] ] if distance_point: additional_fields["_point_of_origin"] = distance_point if raw_result.get("__dist__"): from django.contrib.gis.measure import Distance additional_fields["_distance"] = Distance( km=float(raw_result["__dist__"]) ) else: additional_fields["_distance"] = None result = result_class( app_label, model_name, raw_result[DJANGO_ID], raw_result["score"], **additional_fields ) results.append(result) else: hits -= 1 return { "results": results, "hits": hits, "stats": stats, "facets": facets, "spelling_suggestion": spelling_suggestion, "spelling_suggestions": spelling_suggestions, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest'].get('suggest') if raw_suggest: spelling_suggestion = ' '.join([word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest]) if 'facets' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } for facet_fieldname, facet_info in raw_results['facets'].items(): if facet_info.get('_type', 'terms') == 'terms': facets['fields'][facet_fieldname] = [(individual['term'], individual['count']) for individual in facet_info['terms']] elif facet_info.get('_type', 'terms') == 'date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates'][facet_fieldname] = [(datetime.datetime.utcfromtimestamp(individual['time'] / 1000), individual['count']) for individual in facet_info['entries']] elif facet_info.get('_type', 'terms') == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get('hits', {}).get('hits', []): source = raw_result['_source'] app_label, model_name = source[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result['highlight'].get(content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def get_model(self): if self.is_valid(): model = self.cleaned_data['models'] if model: return haystack_get_model(*model.split(".")) return None
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if hasattr(raw_results, 'stats'): stats = raw_results.stats.get('stats_fields', {}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling and hasattr(raw_results, 'spellcheck'): # Solr 5+ changed the JSON response format so the suggestions will be key-value mapped rather # than simply paired elements in a list, which is a nice improvement but incompatible with # Solr 4: https://issues.apache.org/jira/browse/SOLR-3029 if len(raw_results.spellcheck.get('collations', [])): spelling_suggestion = raw_results.spellcheck['collations'][-1] elif len(raw_results.spellcheck.get('suggestions', [])): spelling_suggestion = raw_results.spellcheck['suggestions'][-1] assert spelling_suggestion is None or isinstance( spelling_suggestion, six.string_types) unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python( value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) del (additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields[ 'highlighted'] = raw_results.highlighting[ raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if hasattr(raw_results, 'stats'): stats = raw_results.stats.get('stats_fields', {}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling is True: if hasattr(raw_results, 'spellcheck'): if len(raw_results.spellcheck.get('suggestions', [])): # For some reason, it's an array of pairs. Pull off the # collated result from the end. spelling_suggestion = raw_results.spellcheck.get( 'suggestions')[-1] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python( value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) del (additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields[ 'highlighted'] = raw_results.highlighting[ raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest'].get('suggest') if raw_suggest: spelling_suggestion = ' '.join([ word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest ]) if 'facets' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } for facet_fieldname, facet_info in raw_results['facets'].items(): if facet_info.get('_type', 'terms') == 'terms': facets['fields'][facet_fieldname] = [ (individual['term'], individual['count']) for individual in facet_info['terms'] ] elif facet_info.get('_type', 'terms') == 'date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets['dates'][facet_fieldname] = [ (datetime.datetime.utcfromtimestamp( individual['time'] / 1000), individual['count']) for individual in facet_info['entries'] ] elif facet_info.get('_type', 'terms') == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get('hits', {}).get('hits', []): source = raw_result['_source'] app_label, model_name = source[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result[ 'highlight'].get(content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results( self, raw_page, highlight=False, query_string="", spelling_query=None, result_class=None, facet_types=None, ): from haystack import connections results = [] # It's important to grab the hits first before slicing. Otherwise, this # can cause pagination failures. hits = len(raw_page) if result_class is None: result_class = SearchResult spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() facets = {} if facet_types: facets = { "fields": {}, "dates": {}, "queries": {}, } for facet_fieldname in raw_page.results.facet_names(): group = raw_page.results.groups(facet_fieldname) facet_type = facet_types[facet_fieldname] # Extract None item for later processing, if present. none_item = group.pop(None, None) lst = facets[facet_type][facet_fieldname] = sorted( group.items(), key=(lambda itm: (-itm[1], itm[0])) ) if none_item is not None: # Inject None item back into the results. none_entry = (None, none_item) if not lst or lst[-1][1] >= none_item: lst.append(none_entry) else: for i, value in enumerate(lst): if value[1] < none_item: lst.insert(i, none_entry) break for doc_offset, raw_result in enumerate(raw_page): score = raw_page.score(doc_offset) or 0 app_label, model_name = raw_result[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in raw_result.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], "convert" ): # Special-cased due to the nature of KEYWORD fields. if index.fields[string_key].is_multivalued: if value is None or len(value) == 0: additional_fields[string_key] = [] else: additional_fields[string_key] = value.split(",") else: additional_fields[string_key] = index.fields[ string_key ].convert(value) else: additional_fields[string_key] = self._to_python(value) del additional_fields[DJANGO_CT] del additional_fields[DJANGO_ID] if highlight: sa = StemmingAnalyzer() formatter = WhooshHtmlFormatter("em") terms = [token.text for token in sa(query_string)] whoosh_result = whoosh_highlight( additional_fields.get(self.content_field_name), terms, sa, ContextFragmenter(), formatter, ) additional_fields["highlighted"] = { self.content_field_name: [whoosh_result] } result = result_class( app_label, model_name, raw_result[DJANGO_ID], score, **additional_fields ) results.append(result) else: hits -= 1 if self.include_spelling: if spelling_query: spelling_suggestion = self.create_spelling_suggestion(spelling_query) else: spelling_suggestion = self.create_spelling_suggestion(query_string) return { "results": results, "hits": hits, "facets": facets, "spelling_suggestion": spelling_suggestion, }
def _process_results( self, raw_results, highlight=False, result_class=None, distance_point=None ): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = spelling_suggestions = None if result_class is None: result_class = SearchResult if hasattr(raw_results, "stats"): stats = raw_results.stats.get("stats_fields", {}) if hasattr(raw_results, "facets"): facets = { "fields": raw_results.facets.get("facet_fields", {}), "dates": raw_results.facets.get("facet_dates", {}), "queries": raw_results.facets.get("facet_queries", {}), } for key in ["fields"]: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip( facets[key][facet_field][::2], facets[key][facet_field][1::2], ) ) if self.include_spelling and hasattr(raw_results, "spellcheck"): try: spelling_suggestions = self.extract_spelling_suggestions(raw_results) except Exception as exc: self.log.error( "Error extracting spelling suggestions: %s", exc, exc_info=True, extra={"data": {"spellcheck": raw_results.spellcheck}}, ) if not self.silently_fail: raise spelling_suggestions = None if spelling_suggestions: # Maintain compatibility with older versions of Haystack which returned a single suggestion: spelling_suggestion = spelling_suggestions[-1] assert isinstance(spelling_suggestion, six.string_types) else: spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], "convert" ): additional_fields[string_key] = index.fields[ string_key ].convert(value) else: additional_fields[string_key] = self.conn._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) del (additional_fields["score"]) if raw_result[ID] in getattr(raw_results, "highlighting", {}): additional_fields["highlighted"] = raw_results.highlighting[ raw_result[ID] ] if distance_point: additional_fields["_point_of_origin"] = distance_point if raw_result.get("__dist__"): from django.contrib.gis.measure import Distance additional_fields["_distance"] = Distance( km=float(raw_result["__dist__"]) ) else: additional_fields["_distance"] = None result = result_class( app_label, model_name, raw_result[DJANGO_ID], raw_result["score"], **additional_fields ) results.append(result) else: hits -= 1 return { "results": results, "hits": hits, "stats": stats, "facets": facets, "spelling_suggestion": spelling_suggestion, "spelling_suggestions": spelling_suggestions, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get('hits', {}).get('total', 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and 'suggest' in raw_results: raw_suggest = raw_results['suggest'].get('suggest') if raw_suggest: spelling_suggestion = ' '.join([ word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest ]) if 'aggregations' in raw_results: facets = { 'fields': {}, 'dates': {}, 'queries': {}, } # ES can return negative timestamps for pre-1970 data. Handle it. def from_timestamp(tm): if tm >= 0: return datetime.utcfromtimestamp(tm) else: return datetime(1970, 1, 1) + timedelta(seconds=tm) for facet_fieldname, facet_info in raw_results[ 'aggregations'].items(): try: facet_type = facet_info['meta']['_type'] except KeyError: facet_type = 'terms' if facet_type == 'terms': facets['fields'][facet_fieldname] = [ (bucket['key'], bucket['doc_count']) for bucket in facet_info['buckets'] ] elif facet_type == 'haystack_date_histogram': # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. dates = [(from_timestamp(bucket['key'] / 1000), bucket['doc_count']) for bucket in facet_info['buckets']] facets['dates'][facet_fieldname[:-len( DATE_HISTOGRAM_FIELD_NAME_SUFFIX)]] = dates elif facet_type == 'haystack_date_range': pass elif facet_type == 'query': facets['queries'][facet_fieldname] = facet_info['count'] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get('hits', {}).get('hits', []): source = raw_result['_source'] app_label, model_name = source[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if 'highlight' in raw_result: additional_fields['highlighted'] = raw_result[ 'highlight'].get(content_field, '') if distance_point: additional_fields['_point_of_origin'] = distance_point if geo_sort and raw_result.get('sort'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['sort'][0])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if hasattr(raw_results,'stats'): stats = raw_results.stats.get('stats_fields',{}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list(zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling and hasattr(raw_results, 'spellcheck'): # Solr 5+ changed the JSON response format so the suggestions will be key-value mapped rather # than simply paired elements in a list, which is a nice improvement but incompatible with # Solr 4: https://issues.apache.org/jira/browse/SOLR-3029 if len(raw_results.spellcheck.get('collations', [])): spelling_suggestion = raw_results.spellcheck['collations'][-1] elif len(raw_results.spellcheck.get('suggestions', [])): spelling_suggestion = raw_results.spellcheck['suggestions'][-1] assert spelling_suggestion is None or isinstance(spelling_suggestion, six.string_types) unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) del(additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_page, highlight=False, query_string='', spelling_query=None, result_class=None): from haystack import connections results = [] # It's important to grab the hits first before slicing. Otherwise, this # can cause pagination failures. hits = len(raw_page) if result_class is None: result_class = SearchResult facets = {} spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for doc_offset, raw_result in enumerate(raw_page): score = raw_page.score(doc_offset) or 0 app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in raw_result.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): # Special-cased due to the nature of KEYWORD fields. if index.fields[string_key].is_multivalued: if value == None or len(value) == 0: additional_fields[string_key] = [] else: additional_fields[string_key] = value.split( ',') else: additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if highlight: sa = StemmingAnalyzer() formatter = WhooshHtmlFormatter('em') terms = [token.text for token in sa(query_string)] whoosh_result = whoosh_highlight( additional_fields.get(self.content_field_name), terms, sa, ContextFragmenter(), formatter) additional_fields['highlighted'] = { self.content_field_name: [whoosh_result], } result = result_class(app_label, model_name, raw_result[DJANGO_ID], score, **additional_fields) results.append(result) else: hits -= 1 if self.include_spelling: if spelling_query: spelling_suggestion = self.create_spelling_suggestion( spelling_query) else: spelling_suggestion = self.create_spelling_suggestion( query_string) return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_page, highlight=False, query_string='', spelling_query=None, result_class=None): from haystack import connections results = [] # It's important to grab the hits first before slicing. Otherwise, this # can cause pagination failures. hits = len(raw_page) if result_class is None: result_class = SearchResult facets = {} spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for doc_offset, raw_result in enumerate(raw_page): score = raw_page.score(doc_offset) or 0 app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in raw_result.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): # Special-cased due to the nature of KEYWORD fields. if index.fields[string_key].is_multivalued: if value is None or len(value) is 0: additional_fields[string_key] = [] else: additional_fields[string_key] = value.split(',') else: additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) if highlight: sa = StemmingAnalyzer() formatter = WhooshHtmlFormatter('em') terms = [token.text for token in sa(query_string)] whoosh_result = whoosh_highlight( additional_fields.get(self.content_field_name), terms, sa, ContextFragmenter(), formatter ) additional_fields['highlighted'] = { self.content_field_name: [whoosh_result], } result = result_class(app_label, model_name, raw_result[DJANGO_ID], score, **additional_fields) results.append(result) else: hits -= 1 if self.include_spelling: if spelling_query: spelling_suggestion = self.create_spelling_suggestion(spelling_query) else: spelling_suggestion = self.create_spelling_suggestion(query_string) return { 'results': results, 'hits': hits, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False): from haystack import connections results = [] hits = raw_results.get("hits", {}).get("total", 0) facets = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if self.include_spelling and "suggest" in raw_results: raw_suggest = raw_results["suggest"].get("suggest") if raw_suggest: spelling_suggestion = " ".join( [word["text"] if len(word["options"]) == 0 else word["options"][0]["text"] for word in raw_suggest] ) if "facets" in raw_results: facets = {"fields": {}, "dates": {}, "queries": {}} for facet_fieldname, facet_info in raw_results["facets"].items(): if facet_info.get("_type", "terms") == "terms": facets["fields"][facet_fieldname] = [ (individual["term"], individual["count"]) for individual in facet_info["terms"] ] elif facet_info.get("_type", "terms") == "date_histogram": # Elasticsearch provides UTC timestamps with an extra three # decimals of precision, which datetime barfs on. facets["dates"][facet_fieldname] = [ (datetime.datetime.utcfromtimestamp(individual["time"] / 1000), individual["count"]) for individual in facet_info["entries"] ] elif facet_info.get("_type", "terms") == "query": facets["queries"][facet_fieldname] = facet_info["count"] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() content_field = unified_index.document_field for raw_result in raw_results.get("hits", {}).get("hits", []): source = raw_result["_source"] app_label, model_name = source[DJANGO_CT].split(".") additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: for key, value in source.items(): index = unified_index.get_index(model) string_key = str(key) if string_key in index.fields and hasattr(index.fields[string_key], "convert"): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self._to_python(value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) if "highlight" in raw_result: additional_fields["highlighted"] = raw_result["highlight"].get(content_field, "") if distance_point: additional_fields["_point_of_origin"] = distance_point if geo_sort and raw_result.get("sort"): from haystack.utils.geo import Distance additional_fields["_distance"] = Distance(km=float(raw_result["sort"][0])) else: additional_fields["_distance"] = None result = result_class( app_label, model_name, source[DJANGO_ID], raw_result["_score"], **additional_fields ) results.append(result) else: hits -= 1 return {"results": results, "hits": hits, "facets": facets, "spelling_suggestion": spelling_suggestion}
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = None if result_class is None: result_class = SearchResult if hasattr(raw_results,'stats'): stats = raw_results.stats.get('stats_fields',{}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list(zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling is True: if hasattr(raw_results, 'spellcheck'): if len(raw_results.spellcheck.get('suggestions', [])): # For some reason, it's an array of pairs. Pull off the # collated result from the end. spelling_suggestion = raw_results.spellcheck.get('suggestions')[-1] unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr(index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python(value) del(additional_fields[DJANGO_CT]) del(additional_fields[DJANGO_ID]) del(additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance(km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, }
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None): from haystack import connections results = [] hits = raw_results.hits facets = {} stats = {} spelling_suggestion = spelling_suggestions = None if result_class is None: result_class = SearchResult if hasattr(raw_results, 'stats'): stats = raw_results.stats.get('stats_fields', {}) if hasattr(raw_results, 'facets'): facets = { 'fields': raw_results.facets.get('facet_fields', {}), 'dates': raw_results.facets.get('facet_dates', {}), 'queries': raw_results.facets.get('facet_queries', {}), } for key in ['fields']: for facet_field in facets[key]: # Convert to a two-tuple, as Solr's json format returns a list of # pairs. facets[key][facet_field] = list( zip(facets[key][facet_field][::2], facets[key][facet_field][1::2])) if self.include_spelling and hasattr(raw_results, 'spellcheck'): try: spelling_suggestions = self.extract_spelling_suggestions( raw_results) except Exception as exc: self.log.error( 'Error extracting spelling suggestions: %s', exc, exc_info=True, extra={'data': { 'spellcheck': raw_results.spellcheck }}) if not self.silently_fail: raise spelling_suggestions = None if spelling_suggestions: # Maintain compatibility with older versions of Haystack which returned a single suggestion: spelling_suggestion = spelling_suggestions[-1] assert isinstance(spelling_suggestion, six.string_types) else: spelling_suggestion = None unified_index = connections[self.connection_alias].get_unified_index() indexed_models = unified_index.get_indexed_models() for raw_result in raw_results.docs: app_label, model_name = raw_result[DJANGO_CT].split('.') additional_fields = {} model = haystack_get_model(app_label, model_name) if model and model in indexed_models: index = unified_index.get_index(model) index_field_map = index.field_map for key, value in raw_result.items(): string_key = str(key) # re-map key if alternate name used if string_key in index_field_map: string_key = index_field_map[key] if string_key in index.fields and hasattr( index.fields[string_key], 'convert'): additional_fields[string_key] = index.fields[ string_key].convert(value) else: additional_fields[string_key] = self.conn._to_python( value) del (additional_fields[DJANGO_CT]) del (additional_fields[DJANGO_ID]) del (additional_fields['score']) if raw_result[ID] in getattr(raw_results, 'highlighting', {}): additional_fields[ 'highlighted'] = raw_results.highlighting[ raw_result[ID]] if distance_point: additional_fields['_point_of_origin'] = distance_point if raw_result.get('__dist__'): from haystack.utils.geo import Distance additional_fields['_distance'] = Distance( km=float(raw_result['__dist__'])) else: additional_fields['_distance'] = None result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields) results.append(result) else: hits -= 1 return { 'results': results, 'hits': hits, 'stats': stats, 'facets': facets, 'spelling_suggestion': spelling_suggestion, 'spelling_suggestions': spelling_suggestions, }