def clear(self, models=None, commit=True):
if models is not None:
assert isinstance(models, (list, tuple))
try:
if models is None:
# *:* matches all docs in Solr
self.conn.delete(q='*:*', commit=commit)
else:
models_to_delete = []
for model in models:
models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model)))
self.conn.delete(q=" OR ".join(models_to_delete), commit=commit)
if commit:
# Run an optimize post-clear. http://wiki.apache.org/solr/FAQ#head-9aafb5d8dff5308e8ea4fcf4b71f19f029c4bb99
self.conn.optimize()
except (IOError, SolrError) as e:
if not self.silently_fail:
raise
if models is not None:
self.log.error("Failed to clear Solr index of models '%s': %s", ','.join(models_to_delete), e,
exc_info=True)
else:
self.log.error("Failed to clear Solr index: %s", e, exc_info=True)
def more_like_this(self, model_instance, additional_query_string=None,
start_offset=0, end_offset=None, models=None,
limit_to_registered_models=None, result_class=None, **kwargs):
from haystack import connections
# Deferred models will have a different class ("RealClass_Deferred_fieldname")
# which won't be in our registry:
model_klass = model_instance._meta.concrete_model
index = connections[self.connection_alias].get_unified_index().get_index(model_klass)
field_name = index.get_content_field()
params = {
'fl': '*,score',
}
if start_offset is not None:
params['start'] = start_offset
if end_offset is not None:
params['rows'] = end_offset
narrow_queries = set()
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
if narrow_queries is None:
narrow_queries = set()
narrow_queries.add('%s:(%s)' % (DJANGO_CT, ' OR '.join(model_choices)))
if additional_query_string:
narrow_queries.add(additional_query_string)
if narrow_queries:
params['fq'] = list(narrow_queries)
query = "%s:%s" % (ID, get_identifier(model_instance))
try:
raw_results = self.conn.more_like_this(query, field_name, **params)
except (IOError, SolrError) as e:
if not self.silently_fail:
raise
self.log.error("Failed to fetch More Like This from Solr for document '%s': %s",
query, e, exc_info=True)
raw_results = EmptyResults()
return self._process_results(raw_results, result_class=result_class)
def clear(self, models=None, commit=True):
if not self.setup_complete:
self.setup()
self.index = self.index.refresh()
if models is not None:
assert isinstance(models, (list, tuple))
try:
if models is None:
self.delete_index()
else:
models_to_delete = []
for model in models:
models_to_delete.append(u"%s:%s" % (DJANGO_CT, get_model_ct(model)))
self.index.delete_by_query(q=self.parser.parse(u" OR ".join(models_to_delete)))
except Exception as e:
if not self.silently_fail:
raise
if models is not None:
self.log.error("Failed to clear Whoosh index of models '%s': %s", ','.join(models_to_delete),
e, exc_info=True)
else:
self.log.error("Failed to clear Whoosh index: %s", e, exc_info=True)
def model_choices(using=DEFAULT_ALIAS):
choices = [
(get_model_ct(m), smart_text(m._meta.verbose_name_plural).title())
for m in connections[using].get_unified_index().get_indexed_models()
if m.__name__ not in EXCLUDE_MODELS
]
return sorted(choices, key=lambda x: x[1])
def build_search_kwargs(self, *args, **kwargs):
kwargs = super(FiasHaystackElasticsearchSearchBackend, self).build_search_kwargs(*args, **kwargs)
query = kwargs.get('query', {}).get('filtered').get('query')
query_string = query.get('query_string', {})
query_string.update({
'default_operator': 'OR',
'auto_generate_phrase_queries': False,
'analyze_wildcard': False,
})
score_script = settings.FIAS_HAYSTACK_ELASTICSEARCH_SCORE_SCRIPT
if score_script:
models = kwargs.get('query', {}).get('filtered', {}).get('filter', {}).get('terms', {}).get('django_ct')
if models and len(models) == 1 and get_model_ct(AddrObj) in models:
kwargs['query'] = {
'function_score': {
'query': kwargs.pop('query'),
'boost_mode': 'replace',
'script_score': {
'script': {
'lang': 'expression',
'inline': score_script,
}
}
}
}
return kwargs
def clear(self, models=None, commit=True):
"""Clear an index.
:param models: Models to be cleared.
:type models: list
:param commit: Commit changes.
:type commit: bool
"""
if models is None:
models = []
doc_type = ''
else:
doc_type = ','.join([get_model_ct(model) for model in models])
try:
if not models:
self.conn.indices.delete(index=self.index_name, ignore=404)
self.setup_complete = False
self.existing_mapping = {}
else:
# Delete by query in Elasticsearch asssumes you're dealing with
# a ``query`` root object. :/
query = {'query': {'query_string': {'query': '*'}}}
self.conn.delete_by_query(index=self.index_name, doc_type=doc_type, body=query)
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
if len(models):
self.log.error("Failed to clear Elasticsearch index of models '%s': %s", doc_type, e)
else:
self.log.error("Failed to clear Elasticsearch index: %s", e)
def clear(self, models=None, commit=True):
# We actually don't want to do this here, as mappings could be
# very different.
# if not self.setup_complete:
# self.setup()
if models is not None:
assert isinstance(models, (list, tuple))
try:
if models is None:
self.conn.indices.delete(index=self.index_name, ignore=404)
self.setup_complete = False
self.existing_mapping = {}
else:
models_to_delete = []
for model in models:
models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model)))
# Delete by query in Elasticsearch asssumes you're dealing with
# a ``query`` root object. :/
query = {'query': {'query_string': {'query': " OR ".join(models_to_delete)}}}
self.conn.delete_by_query(index=self.index_name, doc_type='modelresult', body=query)
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
if models is not None:
self.log.error("Failed to clear Elasticsearch index of models '%s': %s",
','.join(models_to_delete), e, exc_info=True)
else:
self.log.error("Failed to clear Elasticsearch index: %s", e, exc_info=True)
def clear(self, models=None, commit=True):
"""
Clears the backend of all documents/objects for a collection of models.
:param models: List or tuple of models to clear.
:param commit: Not used.
"""
if models is not None:
assert isinstance(models, (list, tuple))
try:
if models is None:
self.conn.indices.delete(index=self.index_name, ignore=404)
self.setup_complete = False
self.existing_mapping = {}
self.content_field_name = None
else:
models_to_delete = []
for model in models:
models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model)))
# Delete using scroll API
query = {
"query": {"query_string": {"query": " OR ".join(models_to_delete)}}
}
generator = scan(
self.conn,
query=query,
index=self.index_name,
doc_type="modelresult",
)
actions = (
{"_op_type": "delete", "_id": doc["_id"]} for doc in generator
)
bulk(
self.conn,
actions=actions,
index=self.index_name,
doc_type="modelresult",
)
self.conn.indices.refresh(index=self.index_name)
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
if models is not None:
self.log.error(
"Failed to clear Elasticsearch index of models '%s': %s",
",".join(models_to_delete),
e,
exc_info=True,
)
else:
self.log.error(
"Failed to clear Elasticsearch index: %s", e, exc_info=True
)
def build_schema(self, indexes):
"""Build Elasticsearch schema.
:param indexes: Dictionary of model -> index.
:type indexes: dict
:return: Schema.
:rtype: dict
"""
schema = {}
for model, index in indexes.iteritems():
mapping_properties = {
DJANGO_CT: {'type': 'string', 'index': 'not_analyzed', 'include_in_all': False},
DJANGO_ID: {'type': 'string', 'index': 'not_analyzed', 'include_in_all': False},
}
for field_name, field_class in index.fields.items():
field_mapping = FIELD_MAPPINGS.get(field_class.field_type, DEFAULT_FIELD_MAPPING).copy()
if field_class.boost != 1.0:
field_mapping['boost'] = field_class.boost
if field_class.stored is True:
field_mapping['store'] = True
# Do this last to override `text` fields.
if field_mapping['type'] == 'string':
if field_class.indexed is False or hasattr(field_class, 'facet_for') or getattr(field_class, 'is_multivalued', False):
field_mapping['index'] = 'not_analyzed'
try:
del field_mapping['analyzer']
del field_mapping['term_vector']
except:
pass
elif field_class.field_type not in ('ngram', 'edge_ngram'):
# Check analyzer attribute
if not hasattr(field_class, 'analyzer') or field_class.analyzer is None:
logger.warning("Set default analyzer for field {}".format(field_name))
field_mapping['index'] = 'analyzed'
field_mapping['analyzer'] = getattr(field_class, 'analyzer', self.DEFAULT_ANALYZER)
# Check term_vector attribute
if hasattr(field_class, 'term_vector') and field_class.term_vector is not None:
field_mapping['term_vector'] = field_class.term_vector
mapping_properties[field_class.index_fieldname] = field_mapping
mapping_type = {
'properties': mapping_properties,
'_boost': {'name': 'boost', 'null_value': 1.0},
}
schema[get_model_ct(model)] = mapping_type
return schema
def update(self, index, iterable, commit=True):
"""Update an index with a collection.
:param index: Index to be updated.
:type index: Index
:param iterable: Objects to update the index.
:type iterable: iterable
:param commit: Commit changes.
:type commit: bool
"""
if not self.setup_complete:
try:
self.setup()
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
self.log.error("Failed to add documents to Elasticsearch: %s", e)
return
prepped_docs = []
for obj in iterable:
try:
prepped_data = index.full_prepare(obj)
final_data = {}
# Convert the data to make sure it's happy.
for key, value in prepped_data.items():
final_data[key] = self._from_python(value)
final_data['_id'] = final_data[ID]
prepped_docs.append(final_data)
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
# We'll log the object identifier but won't include the actual object
# to avoid the possibility of that generating encoding errors while
# processing the log message:
self.log.error(u"%s while preparing object for update" % e.__class__.__name__, exc_info=True, extra={
"data": {
"index": index,
"object": get_identifier(obj)
}
})
doc_type = get_model_ct(index.get_model())
bulk_index(self.conn, prepped_docs, index=self.index_name, doc_type=doc_type)
if commit:
self.conn.indices.refresh(index=self.index_name)
def prepare(self, obj):
"""
Fetches and adds/alters data before indexing.
"""
self.prepared_data = {ID: get_identifier(obj), DJANGO_CT: get_model_ct(obj), DJANGO_ID: force_text(obj.pk)}
for field_name, field in self.fields.items():
# Use the possibly overridden name, which will default to the
# variable name of the field.
self.prepared_data[field.index_fieldname] = field.prepare(obj)
if hasattr(self, "prepare_%s" % field_name):
value = getattr(self, "prepare_%s" % field_name)(obj)
self.prepared_data[field.index_fieldname] = value
return self.prepared_data
def build_models_list(self):
"""
Builds a list of models for searching.
The ``search`` method should use this and the ``django_ct`` field to
narrow the results (unless the user indicates not to). This helps ignore
any results that are not currently handled models and ensures
consistent caching.
"""
from haystack import connections
models = []
for model in connections[self.connection_alias].get_unified_index().get_indexed_models():
models.append(get_model_ct(model))
return models
def get_action_object_content_types():
from rlp.bibliography.models import ProjectReference
from rlp.discussions.models import ThreadedComment
from rlp.documents.models import File, Image, Video, Link
choices = [
('', 'All Types'),
(get_model_ct(ThreadedComment), 'Comments'),
(get_model_ct(File), 'Documents'),
(get_model_ct(Image), 'Images'),
(get_model_ct(Video), 'Videos'),
(get_model_ct(Link), 'Links'),
(get_model_ct(ProjectReference), 'References')
]
return choices
def clear(self, models=[], commit=True):
if not self.setup_complete:
self.setup()
self.index = self.index.refresh()
try:
if not models:
self.delete_index()
else:
models_to_delete = []
for model in models:
models_to_delete.append(u"%s:%s" % (DJANGO_CT, get_model_ct(model)))
self.index.delete_by_query(q=self.parser.parse(u" OR ".join(models_to_delete)))
except Exception as e:
if not self.silently_fail:
raise
self.log.error("Failed to clear documents from Whoosh: %s", e)
def remove(self, obj_or_string, commit=True):
"""Remove an object from an index.
:param obj_or_string: Object to be removed.
:param commit: Commit changes.
:type commit: bool
"""
doc_id = get_identifier(obj_or_string)
try:
doc_type = get_model_ct(obj_or_string)
except:
try:
doc_type = obj_or_string.rsplit('.', 1)[0]
except:
doc_type = '*'
if not self.setup_complete:
try:
self.setup()
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
self.log.error("Failed to remove document '%s' from Elasticsearch: %s", doc_id, e)
return
try:
self.conn.delete(index=self.index_name, doc_type=doc_type, id=doc_id, ignore=404)
if commit:
self.conn.indices.refresh(index=self.index_name)
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
self.log.error("Failed to remove document '%s' from Elasticsearch: %s", doc_id, e)
def clear(self, models=[], commit=True):
try:
if not models:
# *:* matches all docs in Solr
self.conn.delete(q="*:*", commit=commit)
else:
models_to_delete = []
for model in models:
models_to_delete.append("%s:%s" % (DJANGO_CT, get_model_ct(model)))
self.conn.delete(q=" OR ".join(models_to_delete), commit=commit)
if commit:
# Run an optimize post-clear. http://wiki.apache.org/solr/FAQ#head-9aafb5d8dff5308e8ea4fcf4b71f19f029c4bb99
self.conn.optimize()
except (IOError, SolrError) as e:
if not self.silently_fail:
raise
if len(models):
self.log.error("Failed to clear Solr index of models '%s': %s", ",".join(models_to_delete), e)
else:
self.log.error("Failed to clear Solr index: %s", e)
def model_choices(using=DEFAULT_ALIAS):
choices = [
(get_model_ct(m), capfirst(smart_text(m._meta.verbose_name_plural)))
for m in connections[using].get_unified_index().get_indexed_models()
]
return sorted(choices, key=lambda x: x[1])
def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_offset=None,
fields='', highlight=False, facets=None,
date_facets=None, query_facets=None,
narrow_queries=None, spelling_query=None,
within=None, dwithin=None, distance_point=None,
models=None, limit_to_registered_models=None,
result_class=None, stats=None):
kwargs = {'fl': '* score'}
if fields:
if isinstance(fields, (list, set)):
fields = " ".join(fields)
kwargs['fl'] = fields
if sort_by is not None:
if sort_by in ['distance asc', 'distance desc'] and distance_point:
# Do the geo-enabled sort.
lng, lat = distance_point['point'].get_coords()
kwargs['sfield'] = distance_point['field']
kwargs['pt'] = '%s,%s' % (lat, lng)
if sort_by == 'distance asc':
kwargs['sort'] = 'geodist() asc'
else:
kwargs['sort'] = 'geodist() desc'
else:
if sort_by.startswith('distance '):
warnings.warn("In order to sort by distance, you must call the '.distance(...)' method.")
# Regular sorting.
kwargs['sort'] = sort_by
if start_offset is not None:
kwargs['start'] = start_offset
if end_offset is not None:
kwargs['rows'] = end_offset - start_offset
if highlight is True:
kwargs['hl'] = 'true'
kwargs['hl.fragsize'] = '200'
if self.include_spelling is True:
kwargs['spellcheck'] = 'true'
kwargs['spellcheck.collate'] = 'true'
kwargs['spellcheck.count'] = 1
if spelling_query:
kwargs['spellcheck.q'] = spelling_query
if facets is not None:
kwargs['facet'] = 'on'
kwargs['facet.field'] = facets.keys()
for facet_field, options in facets.items():
for key, value in options.items():
kwargs['f.%s.facet.%s' % (facet_field, key)] = self.conn._from_python(value)
if date_facets is not None:
kwargs['facet'] = 'on'
kwargs['facet.date'] = date_facets.keys()
kwargs['facet.date.other'] = 'none'
for key, value in date_facets.items():
kwargs["f.%s.facet.date.start" % key] = self.conn._from_python(value.get('start_date'))
kwargs["f.%s.facet.date.end" % key] = self.conn._from_python(value.get('end_date'))
gap_by_string = value.get('gap_by').upper()
gap_string = "%d%s" % (value.get('gap_amount'), gap_by_string)
if value.get('gap_amount') != 1:
gap_string += "S"
kwargs["f.%s.facet.date.gap" % key] = '+%s/%s' % (gap_string, gap_by_string)
if query_facets is not None:
kwargs['facet'] = 'on'
kwargs['facet.query'] = ["%s:%s" % (field, value) for field, value in query_facets]
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
if narrow_queries is None:
narrow_queries = set()
narrow_queries.add('%s:(%s)' % (DJANGO_CT, ' OR '.join(model_choices)))
if narrow_queries is not None:
kwargs['fq'] = list(narrow_queries)
if stats:
kwargs['stats'] = "true"
for k in stats.keys():
kwargs['stats.field'] = k
for facet in stats[k]:
kwargs['f.%s.stats.facet' % k] = facet
if within is not None:
from haystack.utils.geo import generate_bounding_box
kwargs.setdefault('fq', [])
((min_lat, min_lng), (max_lat, max_lng)) = generate_bounding_box(within['point_1'], within['point_2'])
# Bounding boxes are min, min TO max, max. Solr's wiki was *NOT*
# very clear on this.
bbox = '%s:[%s,%s TO %s,%s]' % (within['field'], min_lat, min_lng, max_lat, max_lng)
kwargs['fq'].append(bbox)
if dwithin is not None:
kwargs.setdefault('fq', [])
lng, lat = dwithin['point'].get_coords()
geofilt = '{!geofilt pt=%s,%s sfield=%s d=%s}' % (lat, lng, dwithin['field'], dwithin['distance'].km)
kwargs['fq'].append(geofilt)
# Check to see if the backend should try to include distances
# (Solr 4.X+) in the results.
if self.distance_available and distance_point:
# In early testing, you can't just hand Solr 4.X a proper bounding box
# & request distances. To enable native distance would take calculating
# a center point & a radius off the user-provided box, which kinda
# sucks. We'll avoid it for now, since Solr 4.x's release will be some
# time yet.
# kwargs['fl'] += ' _dist_:geodist()'
pass
return kwargs
def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_offset=None,
fields='', highlight=False, facets=None,
date_facets=None, query_facets=None,
narrow_queries=None, spelling_query=None,
within=None, dwithin=None, distance_point=None,
models=None, limit_to_registered_models=None,
result_class=None):
index = haystack.connections[self.connection_alias].get_unified_index()
content_field = index.document_field
if query_string == '*:*':
kwargs = {
'query': {
"match_all": {}
},
}
else:
kwargs = {
'query': {
'query_string': {
'default_field': content_field,
'default_operator': DEFAULT_OPERATOR,
'query': query_string,
'analyze_wildcard': True,
'auto_generate_phrase_queries': True,
},
},
}
# so far, no filters
filters = []
if fields:
if isinstance(fields, (list, set)):
fields = " ".join(fields)
kwargs['fields'] = fields
if sort_by is not None:
order_list = []
for field, direction in sort_by:
if field == 'distance' and distance_point:
# Do the geo-enabled sort.
lng, lat = distance_point['point'].get_coords()
sort_kwargs = {
"_geo_distance": {
distance_point['field']: [lng, lat],
"order": direction,
"unit": "km"
}
}
else:
if field == 'distance':
warnings.warn("In order to sort by distance, you must call the '.distance(...)' method.")
# Regular sorting.
sort_kwargs = {field: {'order': direction}}
order_list.append(sort_kwargs)
kwargs['sort'] = order_list
# From/size offsets don't seem to work right in Elasticsearch's DSL. :/
# if start_offset is not None:
# kwargs['from'] = start_offset
# if end_offset is not None:
# kwargs['size'] = end_offset - start_offset
if highlight is True:
kwargs['highlight'] = {
'fields': {
content_field: {'store': 'yes'},
}
}
if self.include_spelling:
kwargs['suggest'] = {
'suggest': {
'text': spelling_query or query_string,
'term': {
# Using content_field here will result in suggestions of stemmed words.
'field': '_all',
},
},
}
if narrow_queries is None:
narrow_queries = set()
if facets is not None:
kwargs.setdefault('facets', {})
for facet_fieldname, extra_options in facets.items():
facet_options = {
'terms': {
'field': facet_fieldname,
'size': 100,
},
}
# Special cases for options applied at the facet level (not the terms level).
if extra_options.pop('global_scope', False):
# Renamed "global_scope" since "global" is a python keyword.
facet_options['global'] = True
if 'facet_filter' in extra_options:
facet_options['facet_filter'] = extra_options.pop('facet_filter')
facet_options['terms'].update(extra_options)
kwargs['facets'][facet_fieldname] = facet_options
if date_facets is not None:
kwargs.setdefault('facets', {})
for facet_fieldname, value in date_facets.items():
# Need to detect on gap_by & only add amount if it's more than one.
interval = value.get('gap_by').lower()
# Need to detect on amount (can't be applied on months or years).
if value.get('gap_amount', 1) != 1 and interval not in ('month', 'year'):
# Just the first character is valid for use.
interval = "%s%s" % (value['gap_amount'], interval[:1])
kwargs['facets'][facet_fieldname] = {
'date_histogram': {
'field': facet_fieldname,
'interval': interval,
},
'facet_filter': {
"range": {
facet_fieldname: {
'from': self._from_python(value.get('start_date')),
'to': self._from_python(value.get('end_date')),
}
}
}
}
if query_facets is not None:
kwargs.setdefault('facets', {})
for facet_fieldname, value in query_facets:
kwargs['facets'][facet_fieldname] = {
'query': {
'query_string': {
'query': value,
}
},
}
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
filters.append({"terms": {DJANGO_CT: model_choices}})
for q in narrow_queries:
filters.append({
'fquery': {
'query': {
'query_string': {
'query': q
},
},
'_cache': True,
}
})
if within is not None:
from haystack.utils.geo import generate_bounding_box
((south, west), (north, east)) = generate_bounding_box(within['point_1'], within['point_2'])
within_filter = {
"geo_bounding_box": {
within['field']: {
"top_left": {
"lat": north,
"lon": west
},
"bottom_right": {
"lat": south,
"lon": east
}
}
},
}
filters.append(within_filter)
if dwithin is not None:
lng, lat = dwithin['point'].get_coords()
# NB: the 1.0.0 release of elasticsearch introduce an
# incompatible change on the distance filter formating
if elasticsearch.VERSION >= (1, 0, 0):
distance = "%(dist).6f%(unit)s" % {
'dist': dwithin['distance'].km,
'unit': "km"
}
else:
distance = dwithin['distance'].km
dwithin_filter = {
"geo_distance": {
"distance": distance,
dwithin['field']: {
"lat": lat,
"lon": lng
}
}
}
filters.append(dwithin_filter)
# if we want to filter, change the query type to filteres
if filters:
kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}}
if len(filters) == 1:
kwargs['query']['filtered']["filter"] = filters[0]
else:
kwargs['query']['filtered']["filter"] = {"bool": {"must": filters}}
return kwargs
def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_offset=None,
fields='', highlight=False, facets=None,
date_facets=None, query_facets=None,
narrow_queries=None, spelling_query=None,
within=None, dwithin=None, distance_point=None,
models=None, limit_to_registered_models=None,
result_class=None):
"""Build all kwargs necessaries to perform the query.
:param query_string: Query string.
:type query_string: str
:param sort_by:
:param start_offset: If query is partially done, this parameters will represents where the slice begins.
:type start_offset: int
:param end_offset: If query is partially done, this parameters will represents where the slice ends.
:type end_offset: int
:param fields: Fields that will be searched for.
:type fields: str
:param highlight:
:param facets:
:param date_facets:
:param query_facets:
:param narrow_queries:
:param spelling_query:
:param within:
:param dwithin:
:param distance_point:
:param models: List of models over the query will be performed.
:type models: list
:param limit_to_registered_models:
:param result_class: Class used for search results.
:type result_class: object
:return: Search kwargs.
:rtype: dict
"""
if query_string == '*:*':
kwargs = {
'query': {
"match_all": {}
},
}
else:
kwargs = {
'query': {
'query_string': {
'default_operator': DEFAULT_OPERATOR,
'query': query_string,
'analyze_wildcard': True,
'auto_generate_phrase_queries': True,
},
},
}
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
kwargs['models'] = model_choices
filters = []
if fields:
if isinstance(fields, (list, set)):
fields = " ".join(fields)
kwargs['fields'] = fields
if sort_by is not None:
order_list = []
for field, direction in sort_by:
if field == 'distance' and distance_point:
# Do the geo-enabled sort.
lng, lat = distance_point['point'].get_coords()
sort_kwargs = {
"_geo_distance": {
distance_point['field']: [lng, lat],
"order": direction,
"unit": "km"
}
}
else:
if field == 'distance':
warnings.warn("In order to sort by distance, you must call the '.distance(...)' method.")
# Regular sorting.
sort_kwargs = {field: {'order': direction}}
order_list.append(sort_kwargs)
kwargs['sort'] = order_list
if highlight is True:
kwargs['highlight'] = {
'fields': {
'_all': {'store': 'yes'},
}
}
if self.include_spelling:
kwargs['suggest'] = {
'suggest': {
'text': spelling_query or query_string,
'term': {
# Using content_field here will result in suggestions of stemmed words.
'field': '_all',
},
},
}
if narrow_queries is None:
narrow_queries = set()
if facets is not None:
kwargs.setdefault('facets', {})
for facet_fieldname, extra_options in facets.items():
facet_options = {
'terms': {
'field': facet_fieldname,
'size': 100,
},
}
# Special cases for options applied at the facet level (not the terms level).
if extra_options.pop('global_scope', False):
# Renamed "global_scope" since "global" is a python keyword.
facet_options['global'] = True
if 'facet_filter' in extra_options:
facet_options['facet_filter'] = extra_options.pop('facet_filter')
facet_options['terms'].update(extra_options)
kwargs['facets'][facet_fieldname] = facet_options
if date_facets is not None:
kwargs.setdefault('facets', {})
for facet_fieldname, value in date_facets.items():
# Need to detect on gap_by & only add amount if it's more than one.
interval = value.get('gap_by').lower()
# Need to detect on amount (can't be applied on months or years).
if value.get('gap_amount', 1) != 1 and interval not in ('month', 'year'):
# Just the first character is valid for use.
interval = "%s%s" % (value['gap_amount'], interval[:1])
kwargs['facets'][facet_fieldname] = {
'date_histogram': {
'field': facet_fieldname,
'interval': interval,
},
'facet_filter': {
"range": {
facet_fieldname: {
'from': self._from_python(value.get('start_date')),
'to': self._from_python(value.get('end_date')),
}
}
}
}
if query_facets is not None:
kwargs.setdefault('facets', {})
for facet_fieldname, value in query_facets:
kwargs['facets'][facet_fieldname] = {
'query': {
'query_string': {
'query': value,
}
},
}
for q in narrow_queries:
filters.append({
'fquery': {
'query': {
'query_string': {
'query': q
},
},
'_cache': True,
}
})
if within is not None:
from haystack.utils.geo import generate_bounding_box
((south, west), (north, east)) = generate_bounding_box(within['point_1'], within['point_2'])
within_filter = {
"geo_bounding_box": {
within['field']: {
"top_left": {
"lat": north,
"lon": west
},
"bottom_right": {
"lat": south,
"lon": east
}
}
},
}
filters.append(within_filter)
if dwithin is not None:
lng, lat = dwithin['point'].get_coords()
dwithin_filter = {
"geo_distance": {
"distance": dwithin['distance'].km,
dwithin['field']: {
"lat": lat,
"lon": lng
}
}
}
filters.append(dwithin_filter)
# if we want to filter, change the query type to filteres
if filters:
kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}}
if len(filters) == 1:
kwargs['query']['filtered']["filter"] = filters[0]
else:
kwargs['query']['filtered']["filter"] = {"bool": {"must": filters}}
return kwargs
def build_search_kwargs(
self,
query_string,
sort_by=None,
start_offset=0,
end_offset=None,
fields="",
highlight=False,
facets=None,
date_facets=None,
query_facets=None,
narrow_queries=None,
spelling_query=None,
within=None,
dwithin=None,
distance_point=None,
models=None,
limit_to_registered_models=None,
result_class=None,
stats=None,
collate=None,
**extra_kwargs
):
index = haystack.connections[self.connection_alias].get_unified_index()
kwargs = {"fl": "* score", "df": index.document_field}
if fields:
if isinstance(fields, (list, set)):
fields = " ".join(fields)
kwargs["fl"] = fields
if sort_by is not None:
if sort_by in ["distance asc", "distance desc"] and distance_point:
# Do the geo-enabled sort.
lng, lat = distance_point["point"].coords
kwargs["sfield"] = distance_point["field"]
kwargs["pt"] = "%s,%s" % (lat, lng)
if sort_by == "distance asc":
kwargs["sort"] = "geodist() asc"
else:
kwargs["sort"] = "geodist() desc"
else:
if sort_by.startswith("distance "):
warnings.warn(
"In order to sort by distance, you must call the '.distance(...)' method."
)
# Regular sorting.
kwargs["sort"] = sort_by
if start_offset is not None:
kwargs["start"] = start_offset
if end_offset is not None:
kwargs["rows"] = end_offset - start_offset
if highlight:
# `highlight` can either be True or a dictionary containing custom parameters
# which will be passed to the backend and may override our default settings:
kwargs["hl"] = "true"
kwargs["hl.fragsize"] = "200"
if isinstance(highlight, dict):
# autoprefix highlighter options with 'hl.', all of them start with it anyway
# this makes option dicts shorter: {'maxAnalyzedChars': 42}
# and lets some of options be used as keyword arguments: `.highlight(preserveMulti=False)`
kwargs.update(
{
key if key.startswith("hl.") else ("hl." + key): highlight[key]
for key in highlight.keys()
}
)
if collate is None:
collate = self.collate
if self.include_spelling is True:
kwargs["spellcheck"] = "true"
kwargs["spellcheck.collate"] = str(collate).lower()
kwargs["spellcheck.count"] = 1
if spelling_query:
kwargs["spellcheck.q"] = spelling_query
if facets is not None:
kwargs["facet"] = "on"
kwargs["facet.field"] = facets.keys()
for facet_field, options in facets.items():
for key, value in options.items():
kwargs[
"f.%s.facet.%s" % (facet_field, key)
] = self.conn._from_python(value)
if date_facets is not None:
kwargs["facet"] = "on"
kwargs["facet.date"] = date_facets.keys()
kwargs["facet.date.other"] = "none"
for key, value in date_facets.items():
kwargs["f.%s.facet.date.start" % key] = self.conn._from_python(
value.get("start_date")
)
kwargs["f.%s.facet.date.end" % key] = self.conn._from_python(
value.get("end_date")
)
gap_by_string = value.get("gap_by").upper()
gap_string = "%d%s" % (value.get("gap_amount"), gap_by_string)
if value.get("gap_amount") != 1:
gap_string += "S"
kwargs["f.%s.facet.date.gap" % key] = "+%s/%s" % (
gap_string,
gap_by_string,
)
if query_facets is not None:
kwargs["facet"] = "on"
kwargs["facet.query"] = [
"%s:%s" % (field, value) for field, value in query_facets
]
if limit_to_registered_models is None:
limit_to_registered_models = getattr(
settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True
)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
if narrow_queries is None:
narrow_queries = set()
narrow_queries.add("%s:(%s)" % (DJANGO_CT, " OR ".join(model_choices)))
if narrow_queries is not None:
kwargs["fq"] = list(narrow_queries)
if stats:
kwargs["stats"] = "true"
for k in stats.keys():
kwargs["stats.field"] = k
for facet in stats[k]:
kwargs["f.%s.stats.facet" % k] = facet
if within is not None:
from haystack.utils.geo import generate_bounding_box
kwargs.setdefault("fq", [])
((min_lat, min_lng), (max_lat, max_lng)) = generate_bounding_box(
within["point_1"], within["point_2"]
)
# Bounding boxes are min, min TO max, max. Solr's wiki was *NOT*
# very clear on this.
bbox = "%s:[%s,%s TO %s,%s]" % (
within["field"],
min_lat,
min_lng,
max_lat,
max_lng,
)
kwargs["fq"].append(bbox)
if dwithin is not None:
kwargs.setdefault("fq", [])
lng, lat = dwithin["point"].coords
geofilt = "{!geofilt pt=%s,%s sfield=%s d=%s}" % (
lat,
lng,
dwithin["field"],
dwithin["distance"].km,
)
kwargs["fq"].append(geofilt)
# Check to see if the backend should try to include distances
# (Solr 4.X+) in the results.
if self.distance_available and distance_point:
# In early testing, you can't just hand Solr 4.X a proper bounding box
# & request distances. To enable native distance would take calculating
# a center point & a radius off the user-provided box, which kinda
# sucks. We'll avoid it for now, since Solr 4.x's release will be some
# time yet.
# kwargs['fl'] += ' _dist_:geodist()'
pass
if extra_kwargs:
kwargs.update(extra_kwargs)
return kwargs
def build_search_kwargs(
self,
query_string,
sort_by=None,
start_offset=0,
end_offset=None,
fields="",
highlight=False,
facets=None,
date_facets=None,
query_facets=None,
narrow_queries=None,
spelling_query=None,
within=None,
dwithin=None,
distance_point=None,
models=None,
limit_to_registered_models=None,
result_class=None,
):
index = haystack.connections[self.connection_alias].get_unified_index()
content_field = index.document_field
if query_string == "*:*":
kwargs = {"query": {"match_all": {}}}
elif query_string.startswith("(") and query_string.endswith(")"):
kwargs = {
"query": {
"query_string": {
"default_field": content_field,
"default_operator": DEFAULT_OPERATOR,
"query": query_string,
"analyze_wildcard": True,
"auto_generate_phrase_queries": True,
}
}
}
else:
kwargs = {
"query": {
"match": {
str(content_field): {
"query": query_string,
"analyzer": self.DEFAULT_ANALYZER, # setting courtesy of ConfigurableElasticBackend
"minimum_should_match": self.DEFAULT_MINIMUM_MATCH,
}
}
}
}
# so far, no filters
filters = []
if fields:
if isinstance(fields, (list, set)):
fields = " ".join(fields)
kwargs["fields"] = fields
if sort_by is not None:
order_list = []
for field, direction in sort_by:
if field == "distance" and distance_point:
# Do the geo-enabled sort.
lng, lat = distance_point["point"].get_coords()
sort_kwargs = {
"_geo_distance": {distance_point["field"]: [lng, lat], "order": direction, "unit": "km"}
}
else:
if field == "distance":
warnings.warn("In order to sort by distance, you must call the '.distance(...)' method.")
# Regular sorting.
sort_kwargs = {field: {"order": direction}}
order_list.append(sort_kwargs)
kwargs["sort"] = order_list
# From/size offsets don't seem to work right in Elasticsearch's DSL. :/
# if start_offset is not None:
# kwargs['from'] = start_offset
# if end_offset is not None:
# kwargs['size'] = end_offset - start_offset
if highlight is True:
kwargs["highlight"] = {"fields": {content_field: {"store": "yes"}}}
if self.include_spelling:
kwargs["suggest"] = {
"suggest": {
"text": spelling_query or query_string,
"term": {
# Using content_field here will result in suggestions of stemmed words.
"field": "_all"
},
}
}
if narrow_queries is None:
narrow_queries = set()
if facets is not None:
kwargs.setdefault("facets", {})
for facet_fieldname, extra_options in facets.items():
facet_options = {"terms": {"field": facet_fieldname, "size": 100}}
# Special cases for options applied at the facet level (not the terms level).
if extra_options.pop("global_scope", False):
# Renamed "global_scope" since "global" is a python keyword.
facet_options["global"] = True
if "facet_filter" in extra_options:
facet_options["facet_filter"] = extra_options.pop("facet_filter")
facet_options["terms"].update(extra_options)
kwargs["facets"][facet_fieldname] = facet_options
if date_facets is not None:
kwargs.setdefault("facets", {})
for facet_fieldname, value in date_facets.items():
# Need to detect on gap_by & only add amount if it's more than one.
interval = value.get("gap_by").lower()
# Need to detect on amount (can't be applied on months or years).
if value.get("gap_amount", 1) != 1 and interval not in ("month", "year"):
# Just the first character is valid for use.
interval = "%s%s" % (value["gap_amount"], interval[:1])
kwargs["facets"][facet_fieldname] = {
"date_histogram": {"field": facet_fieldname, "interval": interval},
"facet_filter": {
"range": {
facet_fieldname: {
"from": self._from_python(value.get("start_date")),
"to": self._from_python(value.get("end_date")),
}
}
},
}
if query_facets is not None:
kwargs.setdefault("facets", {})
for facet_fieldname, value in query_facets:
kwargs["facets"][facet_fieldname] = {"query": {"match": {"query": value}}}
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, "HAYSTACK_LIMIT_TO_REGISTERED_MODELS", True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
filters.append({"terms": {DJANGO_CT: model_choices}})
for q in narrow_queries:
filters.append({"fquery": {"query": {"query_string": {"query": q}}, "_cache": True}})
if within is not None:
from haystack.utils.geo import generate_bounding_box
((south, west), (north, east)) = generate_bounding_box(within["point_1"], within["point_2"])
within_filter = {
"geo_bounding_box": {
within["field"]: {
"top_left": {"lat": north, "lon": west},
"bottom_right": {"lat": south, "lon": east},
}
}
}
filters.append(within_filter)
if dwithin is not None:
lng, lat = dwithin["point"].get_coords()
# NB: the 1.0.0 release of elasticsearch introduce an
# incompatible change on the distance filter formating
if elasticsearch.VERSION >= (1, 0, 0):
distance = "%(dist).6f%(unit)s" % {"dist": dwithin["distance"].km, "unit": "km"}
else:
distance = dwithin["distance"].km
dwithin_filter = {"geo_distance": {"distance": distance, dwithin["field"]: {"lat": lat, "lon": lng}}}
filters.append(dwithin_filter)
# if we want to filter, change the query type to filteres
if filters:
kwargs["query"] = {"filtered": {"query": kwargs.pop("query")}}
if len(filters) == 1:
kwargs["query"]["filtered"]["filter"] = filters[0]
else:
kwargs["query"]["filtered"]["filter"] = {"bool": {"must": filters}}
return kwargs
def search(self, query_string, sort_by=None, start_offset=0, end_offset=None,
fields='', highlight=False, facets=None, date_facets=None, query_facets=None,
narrow_queries=None, spelling_query=None, within=None,
dwithin=None, distance_point=None, models=None,
limit_to_registered_models=None, result_class=None, **kwargs):
if not self.setup_complete:
self.setup()
# A zero length query should return no results.
if len(query_string) == 0:
return {
'results': [],
'hits': 0,
}
query_string = force_text(query_string)
# A one-character query (non-wildcard) gets nabbed by a stopwords
# filter and should yield zero results.
if len(query_string) <= 1 and query_string != u'*':
return {
'results': [],
'hits': 0,
}
reverse = False
if sort_by is not None:
# Determine if we need to reverse the results and if Whoosh can
# handle what it's being asked to sort by. Reversing is an
# all-or-nothing action, unfortunately.
sort_by_list = []
reverse_counter = 0
for order_by in sort_by:
if order_by.startswith('-'):
reverse_counter += 1
if reverse_counter and reverse_counter != len(sort_by):
raise SearchBackendError("Whoosh requires all order_by fields"
" to use the same sort direction")
for order_by in sort_by:
if order_by.startswith('-'):
sort_by_list.append(order_by[1:])
if len(sort_by_list) == 1:
reverse = True
else:
sort_by_list.append(order_by)
if len(sort_by_list) == 1:
reverse = False
sort_by = sort_by_list[0]
if facets is not None:
warnings.warn("Whoosh does not handle faceting.", Warning, stacklevel=2)
if date_facets is not None:
warnings.warn("Whoosh does not handle date faceting.", Warning, stacklevel=2)
if query_facets is not None:
warnings.warn("Whoosh does not handle query faceting.", Warning, stacklevel=2)
narrowed_results = None
self.index = self.index.refresh()
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
if narrow_queries is None:
narrow_queries = set()
narrow_queries.add(' OR '.join(['%s:%s' % (DJANGO_CT, rm) for rm in model_choices]))
narrow_searcher = None
if narrow_queries is not None:
# Potentially expensive? I don't see another way to do it in Whoosh...
narrow_searcher = self.index.searcher()
for nq in narrow_queries:
recent_narrowed_results = narrow_searcher.search(self.parser.parse(force_text(nq)),
limit=None)
if len(recent_narrowed_results) <= 0:
return {
'results': [],
'hits': 0,
}
if narrowed_results:
narrowed_results.filter(recent_narrowed_results)
else:
narrowed_results = recent_narrowed_results
self.index = self.index.refresh()
if self.index.doc_count():
searcher = self.index.searcher()
parsed_query = self.parser.parse(query_string)
# In the event of an invalid/stopworded query, recover gracefully.
if parsed_query is None:
return {
'results': [],
'hits': 0,
}
page_num, page_length = self.calculate_page(start_offset, end_offset)
search_kwargs = {
'pagelen': page_length,
'sortedby': sort_by,
'reverse': reverse,
}
# Handle the case where the results have been narrowed.
if narrowed_results is not None:
search_kwargs['filter'] = narrowed_results
try:
raw_page = searcher.search_page(
parsed_query,
page_num,
**search_kwargs
)
except ValueError:
if not self.silently_fail:
raise
return {
'results': [],
'hits': 0,
'spelling_suggestion': None,
}
# Because as of Whoosh 2.5.1, it will return the wrong page of
# results if you request something too high. :(
if raw_page.pagenum < page_num:
return {
'results': [],
'hits': 0,
'spelling_suggestion': None,
}
results = self._process_results(raw_page, highlight=highlight, query_string=query_string, spelling_query=spelling_query, result_class=result_class)
searcher.close()
if hasattr(narrow_searcher, 'close'):
narrow_searcher.close()
return results
else:
if self.include_spelling:
if spelling_query:
spelling_suggestion = self.create_spelling_suggestion(spelling_query)
else:
spelling_suggestion = self.create_spelling_suggestion(query_string)
else:
spelling_suggestion = None
return {
'results': [],
'hits': 0,
'spelling_suggestion': spelling_suggestion,
}
def more_like_this(self, model_instance, additional_query_string=None,
start_offset=0, end_offset=None, models=None,
limit_to_registered_models=None, result_class=None, **kwargs):
from haystack import connections
if not self.setup_complete:
self.setup()
# Deferred models will have a different class ("RealClass_Deferred_fieldname")
# which won't be in our registry:
model_klass = model_instance._meta.concrete_model
index = connections[self.connection_alias].get_unified_index().get_index(model_klass)
field_name = index.get_content_field()
params = {}
if start_offset is not None:
params['from_'] = start_offset
if end_offset is not None:
params['size'] = end_offset - start_offset
doc_id = get_identifier(model_instance)
try:
# More like this Query
# https://www.elastic.co/guide/en/elasticsearch/reference/2.2/query-dsl-mlt-query.html
mlt_query = {
'query': {
'more_like_this': {
'fields': [field_name],
'like': [{
"_id": doc_id
}]
}
}
}
narrow_queries = []
if additional_query_string and additional_query_string != '*:*':
additional_filter = {
"query": {
"query_string": {
"query": additional_query_string
}
}
}
narrow_queries.append(additional_filter)
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
model_filter = {"terms": {DJANGO_CT: model_choices}}
narrow_queries.append(model_filter)
if len(narrow_queries) > 0:
mlt_query = {
"query": {
"filtered": {
'query': mlt_query['query'],
'filter': {
'bool': {
'must': list(narrow_queries)
}
}
}
}
}
raw_results = self.conn.search(
body=mlt_query,
index=self.index_name,
doc_type='modelresult',
_source=True, **params)
except elasticsearch.TransportError as e:
if not self.silently_fail:
raise
self.log.error("Failed to fetch More Like This from Elasticsearch for document '%s': %s",
doc_id, e, exc_info=True)
raw_results = {}
return self._process_results(raw_results, result_class=result_class)
def models(self, *mods):
# We have to redefine this because Whoosh & Haystack don't play well with model filtering
from haystack.utils import get_model_ct
mods = [get_model_ct(m) for m in mods]
return self.filter(django_ct__in=mods)
def more_like_this(self, model_instance, additional_query_string=None,
start_offset=0, end_offset=None, models=None,
limit_to_registered_models=None, result_class=None, **kwargs):
if not self.setup_complete:
self.setup()
# Deferred models will have a different class ("RealClass_Deferred_fieldname")
# which won't be in our registry:
model_klass = model_instance._meta.concrete_model
field_name = self.content_field_name
narrow_queries = set()
narrowed_results = None
self.index = self.index.refresh()
if limit_to_registered_models is None:
limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
if narrow_queries is None:
narrow_queries = set()
narrow_queries.add(' OR '.join(['%s:%s' % (DJANGO_CT, rm) for rm in model_choices]))
if additional_query_string and additional_query_string != '*':
narrow_queries.add(additional_query_string)
narrow_searcher = None
if narrow_queries is not None:
# Potentially expensive? I don't see another way to do it in Whoosh...
narrow_searcher = self.index.searcher()
for nq in narrow_queries:
recent_narrowed_results = narrow_searcher.search(self.parser.parse(force_text(nq)),
limit=None)
if len(recent_narrowed_results) <= 0:
return {
'results': [],
'hits': 0,
}
if narrowed_results:
narrowed_results.filter(recent_narrowed_results)
else:
narrowed_results = recent_narrowed_results
page_num, page_length = self.calculate_page(start_offset, end_offset)
self.index = self.index.refresh()
raw_results = EmptyResults()
if self.index.doc_count():
query = "%s:%s" % (ID, get_identifier(model_instance))
searcher = self.index.searcher()
parsed_query = self.parser.parse(query)
results = searcher.search(parsed_query)
if len(results):
raw_results = results[0].more_like_this(field_name, top=end_offset)
# Handle the case where the results have been narrowed.
if narrowed_results is not None and hasattr(raw_results, 'filter'):
raw_results.filter(narrowed_results)
try:
raw_page = ResultsPage(raw_results, page_num, page_length)
except ValueError:
if not self.silently_fail:
raise
return {
'results': [],
'hits': 0,
'spelling_suggestion': None,
}
# Because as of Whoosh 2.5.1, it will return the wrong page of
# results if you request something too high. :(
if raw_page.pagenum < page_num:
return {
'results': [],
'hits': 0,
'spelling_suggestion': None,
}
results = self._process_results(raw_page, result_class=result_class)
searcher.close()
if hasattr(narrow_searcher, 'close'):
narrow_searcher.close()
return results
def build_search_kwargs(self,
query_string,
sort_by=None,
start_offset=0,
end_offset=None,
fields='',
highlight=False,
boost_fields=None,
boost_negative=None,
filter_context=None,
narrow_queries=None,
spelling_query=None,
facets=None,
date_facets=None,
query_facets=None,
within=None,
dwithin=None,
distance_point=None,
models=None,
limit_to_registered_models=None,
result_class=None,
**extra_kwargs):
index = haystack.connections[self.connection_alias].get_unified_index()
content_field = index.document_field
filters = []
filters_with_score = []
filter_query_strings = {
'content': u'%s',
'contains': u'*%s*',
'endswith': u'*%s',
'startswith': u'%s*',
'exact': u'%s',
'gt': u'{%s TO *}',
'gte': u'[%s TO *]',
'lt': u'{* TO %s}',
'lte': u'[* TO %s]',
'fuzzy': u'%s~',
}
if filter_context:
for f in filter_context:
if f.get('content'):
content = str(f.pop('content'))
if query_string == '*:*':
query_string = content
else:
query_string = '%s %s' % (query_string, content)
for k, v in f.items():
_filter = None
_filter_with_score = None
try:
_value = v.prepare()
except AttributeError:
_value = str(v)
_field, _lookup = self.get_filter_lookup(k)
_is_nested = NESTED_FILTER_SEPARATOR in _field
_nested_path = None
if _is_nested:
_nested_path = _field.split(NESTED_FILTER_SEPARATOR)[0]
_field = ('.').join(
_field.split(NESTED_FILTER_SEPARATOR))
if _lookup == 'exact':
if _is_nested:
_filter = {'term': {_field: _value}}
else:
_filter = {'term': {_field + '.raw': _value}}
elif _lookup == 'content':
_filter_with_score = {'match': {_field: _value}}
elif _lookup == 'in':
if not isinstance(_value, list):
_value = ast.literal_eval(str(_value))
_filter = {
'query_string': {
'fields': [_field],
'query':
' OR '.join(['"%s"' % i for i in _value])
}
}
elif _lookup == 'range':
if isinstance(_value, dict):
_filter = {'range': {_field: _value}}
elif _value:
if not isinstance(_value, list):
_value = _value.split(',')
if len(_value) >= 2:
_range = {}
_range['gte'] = _value[0]
_range['lte'] = _value[1]
_filter = {'range': {_field: _range}}
else:
raise ValueError(
_('Range lookup requires minimum and maximum values,'
'only one value was provided'))
else:
_filter = {
'query_string': {
'fields': [_field],
'query':
filter_query_strings[_lookup] % _value,
}
}
# nested filter
if _is_nested:
if _filter:
_filter = {
'nested': {
'path': _nested_path,
'query': _filter
}
}
if _filter_with_score:
_filter_with_score = {
'nested': {
'path': _nested_path,
'query': _filter_with_score
}
}
if _filter:
filters.append(_filter)
if _filter_with_score:
filters.append(_filter_with_score)
if query_string == '*:*':
kwargs = {
'query': {
"match_all": {}
},
}
else:
kwargs = {
'query': {
'query_string': {
'fields': [content_field],
'default_operator': DEFAULT_OPERATOR,
'query': query_string,
'analyze_wildcard': True,
'auto_generate_phrase_queries': True,
'fuzzy_max_expansions': FUZZY_MAX_EXPANSIONS,
},
},
}
if boost_fields:
kwargs['query']['query_string']['fields'] = []
for boost_field, boost_value in boost_fields.items():
kwargs['query']['query_string']['fields'].append(
'%s^%s' % (boost_field, boost_value))
if boost_negative:
boosting = {
'positive': kwargs['query'],
'negative': boost_negative[0],
'negative_boost': boost_negative[1]
}
kwargs['query'] = {'boosting': boosting}
if filters_with_score:
kwargs['query'] = {"bool": {"must": [kwargs.pop("query")]}}
kwargs['query']['bool']['must'] += filters_with_score
if fields:
if isinstance(fields, (list, set)):
fields = " ".join(fields)
kwargs['stored_fields'] = fields
if sort_by is not None:
order_list = []
for field, direction in sort_by:
if field == 'distance' and distance_point:
lng, lat = distance_point['point'].get_coords()
sort_kwargs = {
"_geo_distance": {
distance_point['field']: [lng, lat],
"order": direction,
"unit": "km"
}
}
else:
if field == 'distance':
warnings.warn(
"In order to sort by distance, "
"you must call the '.distance(...)' method.")
sort_kwargs = {field: {'order': direction}}
order_list.append(sort_kwargs)
kwargs['sort'] = order_list
if highlight:
kwargs['highlight'] = {
'fields': {
content_field: {},
}
}
if isinstance(highlight, dict):
kwargs['highlight'].update(highlight)
if self.include_spelling:
kwargs['suggest'] = {
'suggest': {
'text': spelling_query or query_string,
'term': {
'field': '_all',
},
},
}
if narrow_queries is None:
narrow_queries = set()
if facets is not None:
kwargs.setdefault('aggregations', {})
for facet_fieldname, extra_options in facets.items():
facet_options = {
'terms': {
'field': facet_fieldname + '.raw',
'size': 100,
},
}
# Special cases for options applied at the facet level (not the terms level).
if extra_options.pop('global_scope', False):
# Renamed "global_scope" since "global" is a python keyword.
facet_options['global'] = True
if 'facet_filter' in extra_options:
facet_options['facet_filter'] = extra_options.pop(
'facet_filter')
facet_options['terms'].update(extra_options)
kwargs['aggregations'][facet_fieldname] = facet_options
if date_facets is not None:
kwargs.setdefault('aggregations', {})
for facet_fieldname, value in date_facets.items():
# Need to detect on gap_by & only add amount if it's more than one.
interval = value.get('gap_by').lower()
# Need to detect on amount (can't be applied on months or years).
if value.get('gap_amount',
1) != 1 and interval not in ('month', 'year'):
# Just the first character is valid for use.
interval = "%s%s" % (value['gap_amount'], interval[:1])
date_histogram_aggregation_name = "{0}{1}".format(
facet_fieldname, DATE_HISTOGRAM_FIELD_NAME_SUFFIX)
date_range_aggregation_name = "{0}{1}".format(
facet_fieldname, DATE_RANGE_FIELD_NAME_SUFFIX)
kwargs['aggregations'][date_histogram_aggregation_name] = {
'meta': {
'_type': 'haystack_date_histogram',
},
'date_histogram': {
'field': facet_fieldname,
'interval': interval,
},
}
kwargs['aggregations'][date_range_aggregation_name] = {
'meta': {
'_type': 'haystack_date_range',
},
'date_range': { # agg type
'field':
facet_fieldname,
'ranges': [{
'from':
self._from_python(value.get('start_date')),
'to':
self._from_python(value.get('end_date')),
}]
}
}
if query_facets is not None:
kwargs.setdefault('aggregations', {})
for facet_fieldname, value in query_facets:
kwargs['aggregations'][facet_fieldname] = {
'filter': {
'query_string': {
'query': value,
}
}
}
if limit_to_registered_models is None:
limit_to_registered_models = getattr(
settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True)
if models and len(models):
model_choices = sorted(get_model_ct(model) for model in models)
elif limit_to_registered_models:
# Using narrow queries, limit the results to only models handled
# with the current routers.
model_choices = self.build_models_list()
else:
model_choices = []
if len(model_choices) > 0:
filters.append({"terms": {DJANGO_CT: model_choices}})
for q in narrow_queries:
filters.append({'query_string': {'query': q}})
if within is not None:
from haystack.utils.geo import generate_bounding_box
((south, west),
(north, east)) = generate_bounding_box(within['point_1'],
within['point_2'])
within_filter = {
"geo_bounding_box": {
within['field']: {
"top_left": {
"lat": north,
"lon": west
},
"bottom_right": {
"lat": south,
"lon": east
}
}
},
}
filters.append(within_filter)
if dwithin is not None:
lng, lat = dwithin['point'].get_coords()
# NB: the 1.0.0 release of elasticsearch introduce an
# incompatible change on the distance filter formating
if elasticsearch.VERSION >= (1, 0, 0):
distance = "%(dist).6f%(unit)s" % {
'dist': dwithin['distance'].km,
'unit': "km"
}
else:
distance = dwithin['distance'].km
dwithin_filter = {
"geo_distance": {
"distance": distance,
dwithin['field']: {
"lat": lat,
"lon": lng
}
}
}
filters.append(dwithin_filter)
# if we want to filter, change the query type to filteres
if filters:
kwargs["query"] = {"bool": {"must": kwargs.pop("query")}}
if len(filters) == 1:
kwargs['query']['bool']["filter"] = filters[0]
else:
kwargs['query']['bool']["filter"] = {"bool": {"must": filters}}
if extra_kwargs:
kwargs.update(extra_kwargs)
return kwargs