def render(self, context):
try:
model_instance = self.model.resolve(context)
sqs = SearchQuerySet()
if self.for_types is not None:
intermediate = template.Variable(self.for_types)
for_types = intermediate.resolve(context).split(",")
search_models = []
for model in for_types:
model_class = haystack_get_model(*model.split("."))
if model_class:
search_models.append(model_class)
sqs = sqs.models(*search_models)
sqs = sqs.more_like_this(model_instance)
if self.limit is not None:
sqs = sqs[: self.limit]
context[self.varname] = sqs
except Exception as exc:
logging.warning(
"Unhandled exception rendering %r: %s", self, exc, exc_info=True
)
return ""
def process_documents(self, doclist, raw_results):
# TODO: tame import spaghetti
from haystack import connections
engine = connections["default"]
conn = engine.get_backend().conn
unified_index = engine.get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in doclist:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in raw_result.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = conn._to_python(value)
del(additional_fields[DJANGO_CT])
del(additional_fields[DJANGO_ID])
del(additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]]
result = SearchResult(app_label, model_name, raw_result[DJANGO_ID],
raw_result['score'], **additional_fields)
yield result
def render(self, context):
try:
model_instance = self.model.resolve(context)
sqs = SearchQuerySet()
if not self.for_types is None:
intermediate = template.Variable(self.for_types)
for_types = intermediate.resolve(context).split(',')
search_models = []
for model in for_types:
model_class = haystack_get_model(*model.split('.'))
if model_class:
search_models.append(model_class)
sqs = sqs.models(*search_models)
sqs = sqs.more_like_this(model_instance)
if not self.limit is None:
sqs = sqs[:self.limit]
context[self.varname] = sqs
except:
pass
return ''
def get_models(self):
"""Return an alphabetical list of model classes in the index."""
search_models = []
if self.is_valid():
for model in self.cleaned_data['models']:
search_models.append(haystack_get_model(*model.split('.')))
return search_models
def get_models(self):
"""Return a list of the selected models."""
search_models = []
if self.is_valid():
for model in self.cleaned_data["models"]:
search_models.append(haystack_get_model(*model.split(".")))
return search_models
def _get_model(self):
if self._model is None:
try:
self._model = haystack_get_model(self.app_label, self.model_name)
except LookupError:
# this changed in change 1.7 to throw an error instead of
# returning None when the model isn't found. So catch the
# lookup error and keep self._model == None.
pass
return self._model
def process_documents(self, doclist):
# TODO: tame import spaghetti
from haystack import connections
engine = connections["default"]
unified_index = engine.get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in doclist:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in raw_result.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[string_key].convert(value)
result = SearchResult(app_label, model_name, raw_result[DJANGO_ID], 1, **additional_fields)
yield result
def _process_results(
self,
raw_results,
highlight=False,
result_class=None,
distance_point=None,
geo_sort=False,
):
from haystack import connections
results = []
hits = raw_results.get("hits", {}).get("total", 0)
facets = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if self.include_spelling and "suggest" in raw_results:
raw_suggest = raw_results["suggest"].get("suggest")
if raw_suggest:
spelling_suggestion = " ".join([
word["text"] if len(word["options"]) == 0 else
word["options"][0]["text"] for word in raw_suggest
])
if "facets" in raw_results:
facets = {"fields": {}, "dates": {}, "queries": {}}
# ES can return negative timestamps for pre-1970 data. Handle it.
def from_timestamp(tm):
if tm >= 0:
return datetime.utcfromtimestamp(tm)
else:
return datetime(1970, 1, 1) + timedelta(seconds=tm)
for facet_fieldname, facet_info in raw_results["facets"].items():
if facet_info.get("_type", "terms") == "terms":
facets["fields"][facet_fieldname] = [
(individual["term"], individual["count"])
for individual in facet_info["terms"]
]
elif facet_info.get("_type", "terms") == "date_histogram":
# Elasticsearch provides UTC timestamps with an extra three
# decimals of precision, which datetime barfs on.
facets["dates"][facet_fieldname] = [
(from_timestamp(individual["time"] / 1000),
individual["count"])
for individual in facet_info["entries"]
]
elif facet_info.get("_type", "terms") == "query":
facets["queries"][facet_fieldname] = facet_info["count"]
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
content_field = unified_index.document_field
for raw_result in raw_results.get("hits", {}).get("hits", []):
source = raw_result["_source"]
app_label, model_name = source[DJANGO_CT].split(".")
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = source and unified_index.get_index(model)
for key, value in source.items():
string_key = str(key)
if string_key in index.fields and hasattr(
index.fields[string_key], "convert"):
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
if "highlight" in raw_result:
additional_fields["highlighted"] = raw_result[
"highlight"].get(content_field, "")
if distance_point:
additional_fields["_point_of_origin"] = distance_point
if geo_sort and raw_result.get("sort"):
from haystack.utils.geo import Distance
additional_fields["_distance"] = Distance(
km=float(raw_result["sort"][0]))
else:
additional_fields["_distance"] = None
result = result_class(app_label, model_name, source[DJANGO_ID],
raw_result["_score"],
**additional_fields)
results.append(result)
else:
hits -= 1
return {
"results": results,
"hits": hits,
"facets": facets,
"spelling_suggestion": spelling_suggestion,
}
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = spelling_suggestions = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results, 'stats'):
stats = raw_results.stats.get('stats_fields', {})
if hasattr(raw_results, 'facets'):
facets = {
'fields': raw_results.facets.get('facet_fields', {}),
'dates': raw_results.facets.get('facet_dates', {}),
'queries': raw_results.facets.get('facet_queries', {}),
}
for key in ['fields']:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(zip(facets[key][facet_field][::2],
facets[key][facet_field][1::2]))
if self.include_spelling and hasattr(raw_results, 'spellcheck'):
try:
spelling_suggestions = self.extract_spelling_suggestions(raw_results)
except Exception as exc:
self.log.error('Error extracting spelling suggestions: %s', exc, exc_info=True,
extra={'data': {'spellcheck': raw_results.spellcheck}})
if not self.silently_fail:
raise
spelling_suggestions = None
if spelling_suggestions:
# Maintain compatibility with older versions of Haystack which returned a single suggestion:
spelling_suggestion = spelling_suggestions[-1]
assert isinstance(spelling_suggestion, six.string_types)
else:
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(value)
del(additional_fields[DJANGO_CT])
del(additional_fields[DJANGO_ID])
del(additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]]
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if raw_result.get('__dist__'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(km=float(raw_result['__dist__']))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'stats': stats,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
'spelling_suggestions': spelling_suggestions,
}
def _process_results(
self, raw_results, highlight=False, result_class=None, distance_point=None
):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = spelling_suggestions = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results, "stats"):
stats = raw_results.stats.get("stats_fields", {})
if hasattr(raw_results, "facets"):
facets = {
"fields": raw_results.facets.get("facet_fields", {}),
"dates": raw_results.facets.get("facet_dates", {}),
"queries": raw_results.facets.get("facet_queries", {}),
"ranges": raw_results.facets.get("facet_ranges", {}),
}
for key in ["fields"]:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(
zip(
facets[key][facet_field][::2],
facets[key][facet_field][1::2],
)
)
for key in ["ranges"]:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(
zip(
facets[key][facet_field]["counts"][::2],
facets[key][facet_field]["counts"][1::2],
)
)
if self.include_spelling and hasattr(raw_results, "spellcheck"):
try:
spelling_suggestions = self.extract_spelling_suggestions(raw_results)
except Exception as exc:
self.log.error(
"Error extracting spelling suggestions: %s",
exc,
exc_info=True,
extra={"data": {"spellcheck": raw_results.spellcheck}},
)
if not self.silently_fail:
raise
spelling_suggestions = None
if spelling_suggestions:
# Maintain compatibility with older versions of Haystack which returned a single suggestion:
spelling_suggestion = spelling_suggestions[-1]
assert isinstance(spelling_suggestion, str)
else:
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split(".")
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(
index.fields[string_key], "convert"
):
additional_fields[string_key] = index.fields[
string_key
].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(value)
del additional_fields[DJANGO_CT]
del additional_fields[DJANGO_ID]
del additional_fields["score"]
if raw_result[ID] in getattr(raw_results, "highlighting", {}):
additional_fields["highlighted"] = raw_results.highlighting[
raw_result[ID]
]
if distance_point:
additional_fields["_point_of_origin"] = distance_point
if raw_result.get("__dist__"):
from django.contrib.gis.measure import Distance
additional_fields["_distance"] = Distance(
km=float(raw_result["__dist__"])
)
else:
additional_fields["_distance"] = None
result = result_class(
app_label,
model_name,
raw_result[DJANGO_ID],
raw_result["score"],
**additional_fields
)
results.append(result)
else:
hits -= 1
return {
"results": results,
"hits": hits,
"stats": stats,
"facets": facets,
"spelling_suggestion": spelling_suggestion,
"spelling_suggestions": spelling_suggestions,
}
def _process_results(self, raw_results, highlight=False,
result_class=None, distance_point=None,
geo_sort=False):
from haystack import connections
results = []
hits = raw_results.get('hits', {}).get('total', 0)
facets = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if self.include_spelling and 'suggest' in raw_results:
raw_suggest = raw_results['suggest'].get('suggest')
if raw_suggest:
spelling_suggestion = ' '.join([word['text'] if len(word['options']) == 0 else word['options'][0]['text'] for word in raw_suggest])
if 'facets' in raw_results:
facets = {
'fields': {},
'dates': {},
'queries': {},
}
for facet_fieldname, facet_info in raw_results['facets'].items():
if facet_info.get('_type', 'terms') == 'terms':
facets['fields'][facet_fieldname] = [(individual['term'], individual['count']) for individual in facet_info['terms']]
elif facet_info.get('_type', 'terms') == 'date_histogram':
# Elasticsearch provides UTC timestamps with an extra three
# decimals of precision, which datetime barfs on.
facets['dates'][facet_fieldname] = [(datetime.datetime.utcfromtimestamp(individual['time'] / 1000), individual['count']) for individual in facet_info['entries']]
elif facet_info.get('_type', 'terms') == 'query':
facets['queries'][facet_fieldname] = facet_info['count']
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
content_field = unified_index.document_field
for raw_result in raw_results.get('hits', {}).get('hits', []):
source = raw_result['_source']
app_label, model_name = source[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in source.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del(additional_fields[DJANGO_CT])
del(additional_fields[DJANGO_ID])
if 'highlight' in raw_result:
additional_fields['highlighted'] = raw_result['highlight'].get(content_field, '')
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if geo_sort and raw_result.get('sort'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(km=float(raw_result['sort'][0]))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name, source[DJANGO_ID], raw_result['_score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def get_model(self):
if self.is_valid():
model = self.cleaned_data['models']
if model:
return haystack_get_model(*model.split("."))
return None
def _process_results(self,
raw_results,
highlight=False,
result_class=None,
distance_point=None):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results, 'stats'):
stats = raw_results.stats.get('stats_fields', {})
if hasattr(raw_results, 'facets'):
facets = {
'fields': raw_results.facets.get('facet_fields', {}),
'dates': raw_results.facets.get('facet_dates', {}),
'queries': raw_results.facets.get('facet_queries', {}),
}
for key in ['fields']:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(
zip(facets[key][facet_field][::2],
facets[key][facet_field][1::2]))
if self.include_spelling and hasattr(raw_results, 'spellcheck'):
# Solr 5+ changed the JSON response format so the suggestions will be key-value mapped rather
# than simply paired elements in a list, which is a nice improvement but incompatible with
# Solr 4: https://issues.apache.org/jira/browse/SOLR-3029
if len(raw_results.spellcheck.get('collations', [])):
spelling_suggestion = raw_results.spellcheck['collations'][-1]
elif len(raw_results.spellcheck.get('suggestions', [])):
spelling_suggestion = raw_results.spellcheck['suggestions'][-1]
assert spelling_suggestion is None or isinstance(
spelling_suggestion, six.string_types)
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(
index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(
value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
del (additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields[
'highlighted'] = raw_results.highlighting[
raw_result[ID]]
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if raw_result.get('__dist__'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(
km=float(raw_result['__dist__']))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name,
raw_result[DJANGO_ID],
raw_result['score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'stats': stats,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self,
raw_results,
highlight=False,
result_class=None,
distance_point=None):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results, 'stats'):
stats = raw_results.stats.get('stats_fields', {})
if hasattr(raw_results, 'facets'):
facets = {
'fields': raw_results.facets.get('facet_fields', {}),
'dates': raw_results.facets.get('facet_dates', {}),
'queries': raw_results.facets.get('facet_queries', {}),
}
for key in ['fields']:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(
zip(facets[key][facet_field][::2],
facets[key][facet_field][1::2]))
if self.include_spelling is True:
if hasattr(raw_results, 'spellcheck'):
if len(raw_results.spellcheck.get('suggestions', [])):
# For some reason, it's an array of pairs. Pull off the
# collated result from the end.
spelling_suggestion = raw_results.spellcheck.get(
'suggestions')[-1]
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(
index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(
value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
del (additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields[
'highlighted'] = raw_results.highlighting[
raw_result[ID]]
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if raw_result.get('__dist__'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(
km=float(raw_result['__dist__']))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name,
raw_result[DJANGO_ID],
raw_result['score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'stats': stats,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self,
raw_results,
highlight=False,
result_class=None,
distance_point=None,
geo_sort=False):
from haystack import connections
results = []
hits = raw_results.get('hits', {}).get('total', 0)
facets = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if self.include_spelling and 'suggest' in raw_results:
raw_suggest = raw_results['suggest'].get('suggest')
if raw_suggest:
spelling_suggestion = ' '.join([
word['text'] if len(word['options']) == 0 else
word['options'][0]['text'] for word in raw_suggest
])
if 'facets' in raw_results:
facets = {
'fields': {},
'dates': {},
'queries': {},
}
for facet_fieldname, facet_info in raw_results['facets'].items():
if facet_info.get('_type', 'terms') == 'terms':
facets['fields'][facet_fieldname] = [
(individual['term'], individual['count'])
for individual in facet_info['terms']
]
elif facet_info.get('_type', 'terms') == 'date_histogram':
# Elasticsearch provides UTC timestamps with an extra three
# decimals of precision, which datetime barfs on.
facets['dates'][facet_fieldname] = [
(datetime.datetime.utcfromtimestamp(
individual['time'] / 1000), individual['count'])
for individual in facet_info['entries']
]
elif facet_info.get('_type', 'terms') == 'query':
facets['queries'][facet_fieldname] = facet_info['count']
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
content_field = unified_index.document_field
for raw_result in raw_results.get('hits', {}).get('hits', []):
source = raw_result['_source']
app_label, model_name = source[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in source.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(
index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
if 'highlight' in raw_result:
additional_fields['highlighted'] = raw_result[
'highlight'].get(content_field, '')
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if geo_sort and raw_result.get('sort'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(
km=float(raw_result['sort'][0]))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name, source[DJANGO_ID],
raw_result['_score'],
**additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(
self,
raw_page,
highlight=False,
query_string="",
spelling_query=None,
result_class=None,
facet_types=None,
):
from haystack import connections
results = []
# It's important to grab the hits first before slicing. Otherwise, this
# can cause pagination failures.
hits = len(raw_page)
if result_class is None:
result_class = SearchResult
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
facets = {}
if facet_types:
facets = {
"fields": {},
"dates": {},
"queries": {},
}
for facet_fieldname in raw_page.results.facet_names():
group = raw_page.results.groups(facet_fieldname)
facet_type = facet_types[facet_fieldname]
# Extract None item for later processing, if present.
none_item = group.pop(None, None)
lst = facets[facet_type][facet_fieldname] = sorted(
group.items(), key=(lambda itm: (-itm[1], itm[0]))
)
if none_item is not None:
# Inject None item back into the results.
none_entry = (None, none_item)
if not lst or lst[-1][1] >= none_item:
lst.append(none_entry)
else:
for i, value in enumerate(lst):
if value[1] < none_item:
lst.insert(i, none_entry)
break
for doc_offset, raw_result in enumerate(raw_page):
score = raw_page.score(doc_offset) or 0
app_label, model_name = raw_result[DJANGO_CT].split(".")
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in raw_result.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(
index.fields[string_key], "convert"
):
# Special-cased due to the nature of KEYWORD fields.
if index.fields[string_key].is_multivalued:
if value is None or len(value) == 0:
additional_fields[string_key] = []
else:
additional_fields[string_key] = value.split(",")
else:
additional_fields[string_key] = index.fields[
string_key
].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del additional_fields[DJANGO_CT]
del additional_fields[DJANGO_ID]
if highlight:
sa = StemmingAnalyzer()
formatter = WhooshHtmlFormatter("em")
terms = [token.text for token in sa(query_string)]
whoosh_result = whoosh_highlight(
additional_fields.get(self.content_field_name),
terms,
sa,
ContextFragmenter(),
formatter,
)
additional_fields["highlighted"] = {
self.content_field_name: [whoosh_result]
}
result = result_class(
app_label,
model_name,
raw_result[DJANGO_ID],
score,
**additional_fields
)
results.append(result)
else:
hits -= 1
if self.include_spelling:
if spelling_query:
spelling_suggestion = self.create_spelling_suggestion(spelling_query)
else:
spelling_suggestion = self.create_spelling_suggestion(query_string)
return {
"results": results,
"hits": hits,
"facets": facets,
"spelling_suggestion": spelling_suggestion,
}
def _process_results(
self, raw_results, highlight=False, result_class=None, distance_point=None
):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = spelling_suggestions = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results, "stats"):
stats = raw_results.stats.get("stats_fields", {})
if hasattr(raw_results, "facets"):
facets = {
"fields": raw_results.facets.get("facet_fields", {}),
"dates": raw_results.facets.get("facet_dates", {}),
"queries": raw_results.facets.get("facet_queries", {}),
}
for key in ["fields"]:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(
zip(
facets[key][facet_field][::2],
facets[key][facet_field][1::2],
)
)
if self.include_spelling and hasattr(raw_results, "spellcheck"):
try:
spelling_suggestions = self.extract_spelling_suggestions(raw_results)
except Exception as exc:
self.log.error(
"Error extracting spelling suggestions: %s",
exc,
exc_info=True,
extra={"data": {"spellcheck": raw_results.spellcheck}},
)
if not self.silently_fail:
raise
spelling_suggestions = None
if spelling_suggestions:
# Maintain compatibility with older versions of Haystack which returned a single suggestion:
spelling_suggestion = spelling_suggestions[-1]
assert isinstance(spelling_suggestion, six.string_types)
else:
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split(".")
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(
index.fields[string_key], "convert"
):
additional_fields[string_key] = index.fields[
string_key
].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
del (additional_fields["score"])
if raw_result[ID] in getattr(raw_results, "highlighting", {}):
additional_fields["highlighted"] = raw_results.highlighting[
raw_result[ID]
]
if distance_point:
additional_fields["_point_of_origin"] = distance_point
if raw_result.get("__dist__"):
from django.contrib.gis.measure import Distance
additional_fields["_distance"] = Distance(
km=float(raw_result["__dist__"])
)
else:
additional_fields["_distance"] = None
result = result_class(
app_label,
model_name,
raw_result[DJANGO_ID],
raw_result["score"],
**additional_fields
)
results.append(result)
else:
hits -= 1
return {
"results": results,
"hits": hits,
"stats": stats,
"facets": facets,
"spelling_suggestion": spelling_suggestion,
"spelling_suggestions": spelling_suggestions,
}
def _process_results(self,
raw_results,
highlight=False,
result_class=None,
distance_point=None,
geo_sort=False):
from haystack import connections
results = []
hits = raw_results.get('hits', {}).get('total', 0)
facets = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if self.include_spelling and 'suggest' in raw_results:
raw_suggest = raw_results['suggest'].get('suggest')
if raw_suggest:
spelling_suggestion = ' '.join([
word['text'] if len(word['options']) == 0 else
word['options'][0]['text'] for word in raw_suggest
])
if 'aggregations' in raw_results:
facets = {
'fields': {},
'dates': {},
'queries': {},
}
# ES can return negative timestamps for pre-1970 data. Handle it.
def from_timestamp(tm):
if tm >= 0:
return datetime.utcfromtimestamp(tm)
else:
return datetime(1970, 1, 1) + timedelta(seconds=tm)
for facet_fieldname, facet_info in raw_results[
'aggregations'].items():
try:
facet_type = facet_info['meta']['_type']
except KeyError:
facet_type = 'terms'
if facet_type == 'terms':
facets['fields'][facet_fieldname] = [
(bucket['key'], bucket['doc_count'])
for bucket in facet_info['buckets']
]
elif facet_type == 'haystack_date_histogram':
# Elasticsearch provides UTC timestamps with an extra three
# decimals of precision, which datetime barfs on.
dates = [(from_timestamp(bucket['key'] / 1000),
bucket['doc_count'])
for bucket in facet_info['buckets']]
facets['dates'][facet_fieldname[:-len(
DATE_HISTOGRAM_FIELD_NAME_SUFFIX)]] = dates
elif facet_type == 'haystack_date_range':
pass
elif facet_type == 'query':
facets['queries'][facet_fieldname] = facet_info['count']
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
content_field = unified_index.document_field
for raw_result in raw_results.get('hits', {}).get('hits', []):
source = raw_result['_source']
app_label, model_name = source[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in source.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(
index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
if 'highlight' in raw_result:
additional_fields['highlighted'] = raw_result[
'highlight'].get(content_field, '')
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if geo_sort and raw_result.get('sort'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(
km=float(raw_result['sort'][0]))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name, source[DJANGO_ID],
raw_result['_score'],
**additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results,'stats'):
stats = raw_results.stats.get('stats_fields',{})
if hasattr(raw_results, 'facets'):
facets = {
'fields': raw_results.facets.get('facet_fields', {}),
'dates': raw_results.facets.get('facet_dates', {}),
'queries': raw_results.facets.get('facet_queries', {}),
}
for key in ['fields']:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(zip(facets[key][facet_field][::2], facets[key][facet_field][1::2]))
if self.include_spelling and hasattr(raw_results, 'spellcheck'):
# Solr 5+ changed the JSON response format so the suggestions will be key-value mapped rather
# than simply paired elements in a list, which is a nice improvement but incompatible with
# Solr 4: https://issues.apache.org/jira/browse/SOLR-3029
if len(raw_results.spellcheck.get('collations', [])):
spelling_suggestion = raw_results.spellcheck['collations'][-1]
elif len(raw_results.spellcheck.get('suggestions', [])):
spelling_suggestion = raw_results.spellcheck['suggestions'][-1]
assert spelling_suggestion is None or isinstance(spelling_suggestion, six.string_types)
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(value)
del(additional_fields[DJANGO_CT])
del(additional_fields[DJANGO_ID])
del(additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]]
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if raw_result.get('__dist__'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(km=float(raw_result['__dist__']))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'stats': stats,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self,
raw_page,
highlight=False,
query_string='',
spelling_query=None,
result_class=None):
from haystack import connections
results = []
# It's important to grab the hits first before slicing. Otherwise, this
# can cause pagination failures.
hits = len(raw_page)
if result_class is None:
result_class = SearchResult
facets = {}
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for doc_offset, raw_result in enumerate(raw_page):
score = raw_page.score(doc_offset) or 0
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in raw_result.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(
index.fields[string_key], 'convert'):
# Special-cased due to the nature of KEYWORD fields.
if index.fields[string_key].is_multivalued:
if value == None or len(value) == 0:
additional_fields[string_key] = []
else:
additional_fields[string_key] = value.split(
',')
else:
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
if highlight:
sa = StemmingAnalyzer()
formatter = WhooshHtmlFormatter('em')
terms = [token.text for token in sa(query_string)]
whoosh_result = whoosh_highlight(
additional_fields.get(self.content_field_name), terms,
sa, ContextFragmenter(), formatter)
additional_fields['highlighted'] = {
self.content_field_name: [whoosh_result],
}
result = result_class(app_label, model_name,
raw_result[DJANGO_ID], score,
**additional_fields)
results.append(result)
else:
hits -= 1
if self.include_spelling:
if spelling_query:
spelling_suggestion = self.create_spelling_suggestion(
spelling_query)
else:
spelling_suggestion = self.create_spelling_suggestion(
query_string)
return {
'results': results,
'hits': hits,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self, raw_page, highlight=False, query_string='', spelling_query=None, result_class=None):
from haystack import connections
results = []
# It's important to grab the hits first before slicing. Otherwise, this
# can cause pagination failures.
hits = len(raw_page)
if result_class is None:
result_class = SearchResult
facets = {}
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for doc_offset, raw_result in enumerate(raw_page):
score = raw_page.score(doc_offset) or 0
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in raw_result.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
# Special-cased due to the nature of KEYWORD fields.
if index.fields[string_key].is_multivalued:
if value is None or len(value) is 0:
additional_fields[string_key] = []
else:
additional_fields[string_key] = value.split(',')
else:
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del(additional_fields[DJANGO_CT])
del(additional_fields[DJANGO_ID])
if highlight:
sa = StemmingAnalyzer()
formatter = WhooshHtmlFormatter('em')
terms = [token.text for token in sa(query_string)]
whoosh_result = whoosh_highlight(
additional_fields.get(self.content_field_name),
terms,
sa,
ContextFragmenter(),
formatter
)
additional_fields['highlighted'] = {
self.content_field_name: [whoosh_result],
}
result = result_class(app_label, model_name, raw_result[DJANGO_ID], score, **additional_fields)
results.append(result)
else:
hits -= 1
if self.include_spelling:
if spelling_query:
spelling_suggestion = self.create_spelling_suggestion(spelling_query)
else:
spelling_suggestion = self.create_spelling_suggestion(query_string)
return {
'results': results,
'hits': hits,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None, geo_sort=False):
from haystack import connections
results = []
hits = raw_results.get("hits", {}).get("total", 0)
facets = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if self.include_spelling and "suggest" in raw_results:
raw_suggest = raw_results["suggest"].get("suggest")
if raw_suggest:
spelling_suggestion = " ".join(
[word["text"] if len(word["options"]) == 0 else word["options"][0]["text"] for word in raw_suggest]
)
if "facets" in raw_results:
facets = {"fields": {}, "dates": {}, "queries": {}}
for facet_fieldname, facet_info in raw_results["facets"].items():
if facet_info.get("_type", "terms") == "terms":
facets["fields"][facet_fieldname] = [
(individual["term"], individual["count"]) for individual in facet_info["terms"]
]
elif facet_info.get("_type", "terms") == "date_histogram":
# Elasticsearch provides UTC timestamps with an extra three
# decimals of precision, which datetime barfs on.
facets["dates"][facet_fieldname] = [
(datetime.datetime.utcfromtimestamp(individual["time"] / 1000), individual["count"])
for individual in facet_info["entries"]
]
elif facet_info.get("_type", "terms") == "query":
facets["queries"][facet_fieldname] = facet_info["count"]
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
content_field = unified_index.document_field
for raw_result in raw_results.get("hits", {}).get("hits", []):
source = raw_result["_source"]
app_label, model_name = source[DJANGO_CT].split(".")
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
for key, value in source.items():
index = unified_index.get_index(model)
string_key = str(key)
if string_key in index.fields and hasattr(index.fields[string_key], "convert"):
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = self._to_python(value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
if "highlight" in raw_result:
additional_fields["highlighted"] = raw_result["highlight"].get(content_field, "")
if distance_point:
additional_fields["_point_of_origin"] = distance_point
if geo_sort and raw_result.get("sort"):
from haystack.utils.geo import Distance
additional_fields["_distance"] = Distance(km=float(raw_result["sort"][0]))
else:
additional_fields["_distance"] = None
result = result_class(
app_label, model_name, source[DJANGO_ID], raw_result["_score"], **additional_fields
)
results.append(result)
else:
hits -= 1
return {"results": results, "hits": hits, "facets": facets, "spelling_suggestion": spelling_suggestion}
def _process_results(self, raw_results, highlight=False, result_class=None, distance_point=None):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results,'stats'):
stats = raw_results.stats.get('stats_fields',{})
if hasattr(raw_results, 'facets'):
facets = {
'fields': raw_results.facets.get('facet_fields', {}),
'dates': raw_results.facets.get('facet_dates', {}),
'queries': raw_results.facets.get('facet_queries', {}),
}
for key in ['fields']:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(zip(facets[key][facet_field][::2], facets[key][facet_field][1::2]))
if self.include_spelling is True:
if hasattr(raw_results, 'spellcheck'):
if len(raw_results.spellcheck.get('suggestions', [])):
# For some reason, it's an array of pairs. Pull off the
# collated result from the end.
spelling_suggestion = raw_results.spellcheck.get('suggestions')[-1]
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[string_key].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(value)
del(additional_fields[DJANGO_CT])
del(additional_fields[DJANGO_ID])
del(additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields['highlighted'] = raw_results.highlighting[raw_result[ID]]
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if raw_result.get('__dist__'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(km=float(raw_result['__dist__']))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name, raw_result[DJANGO_ID], raw_result['score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'stats': stats,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
}
def _process_results(self,
raw_results,
highlight=False,
result_class=None,
distance_point=None):
from haystack import connections
results = []
hits = raw_results.hits
facets = {}
stats = {}
spelling_suggestion = spelling_suggestions = None
if result_class is None:
result_class = SearchResult
if hasattr(raw_results, 'stats'):
stats = raw_results.stats.get('stats_fields', {})
if hasattr(raw_results, 'facets'):
facets = {
'fields': raw_results.facets.get('facet_fields', {}),
'dates': raw_results.facets.get('facet_dates', {}),
'queries': raw_results.facets.get('facet_queries', {}),
}
for key in ['fields']:
for facet_field in facets[key]:
# Convert to a two-tuple, as Solr's json format returns a list of
# pairs.
facets[key][facet_field] = list(
zip(facets[key][facet_field][::2],
facets[key][facet_field][1::2]))
if self.include_spelling and hasattr(raw_results, 'spellcheck'):
try:
spelling_suggestions = self.extract_spelling_suggestions(
raw_results)
except Exception as exc:
self.log.error(
'Error extracting spelling suggestions: %s',
exc,
exc_info=True,
extra={'data': {
'spellcheck': raw_results.spellcheck
}})
if not self.silently_fail:
raise
spelling_suggestions = None
if spelling_suggestions:
# Maintain compatibility with older versions of Haystack which returned a single suggestion:
spelling_suggestion = spelling_suggestions[-1]
assert isinstance(spelling_suggestion, six.string_types)
else:
spelling_suggestion = None
unified_index = connections[self.connection_alias].get_unified_index()
indexed_models = unified_index.get_indexed_models()
for raw_result in raw_results.docs:
app_label, model_name = raw_result[DJANGO_CT].split('.')
additional_fields = {}
model = haystack_get_model(app_label, model_name)
if model and model in indexed_models:
index = unified_index.get_index(model)
index_field_map = index.field_map
for key, value in raw_result.items():
string_key = str(key)
# re-map key if alternate name used
if string_key in index_field_map:
string_key = index_field_map[key]
if string_key in index.fields and hasattr(
index.fields[string_key], 'convert'):
additional_fields[string_key] = index.fields[
string_key].convert(value)
else:
additional_fields[string_key] = self.conn._to_python(
value)
del (additional_fields[DJANGO_CT])
del (additional_fields[DJANGO_ID])
del (additional_fields['score'])
if raw_result[ID] in getattr(raw_results, 'highlighting', {}):
additional_fields[
'highlighted'] = raw_results.highlighting[
raw_result[ID]]
if distance_point:
additional_fields['_point_of_origin'] = distance_point
if raw_result.get('__dist__'):
from haystack.utils.geo import Distance
additional_fields['_distance'] = Distance(
km=float(raw_result['__dist__']))
else:
additional_fields['_distance'] = None
result = result_class(app_label, model_name,
raw_result[DJANGO_ID],
raw_result['score'], **additional_fields)
results.append(result)
else:
hits -= 1
return {
'results': results,
'hits': hits,
'stats': stats,
'facets': facets,
'spelling_suggestion': spelling_suggestion,
'spelling_suggestions': spelling_suggestions,
}
