def search(self):
sqs = SearchQuerySet().load_all()
if not self.is_valid():
return sqs
if self.cleaned_data['q']:
# search input of a code, contains at least '_YYYY_'
# if not re.match('([a-zA-Z]*_?[EI])?_[0-9]{4}_([0-9]{3}_[0-9]{3})?', self.cleaned_data.get('q')):
sqs = sqs.filter(content__startswith=self.cleaned_data['q']).facet(
'item_acces').facet('item_status').facet('digitized').facet(
'recording_context').facet('physical_format').facet(
'media_type')
# else:
# sqs = sqs.filter(code__contains=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type')
for facet in self.selected_facets:
if ":" not in facet:
continue
field, value = facet.split(":", 1)
if value:
if value == 'viewable':
sqs = sqs.narrow('item_acces:full OR item_acces:mixed'
).narrow('digitized:T')
else:
sqs = sqs.narrow(u'%s:"%s"' %
(field, sqs.query.clean(value)))
return sqs
def search(self):
sqs = SearchQuerySet().load_all()
if not self.is_valid():
return sqs
if self.cleaned_data['q']:
#search input of a code, contains at least '_YYYY_'
#if not re.match('([a-zA-Z]*_?[EI])?_[0-9]{4}_([0-9]{3}_[0-9]{3})?', self.cleaned_data.get('q')):
sqs = sqs.filter(content__startswith=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type')
#else:
# sqs = sqs.filter(code__contains=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type')
for facet in self.selected_facets:
if ":" not in facet:
continue
field, value = facet.split(":", 1)
if value:
if value == 'viewable':
sqs = sqs.narrow('item_acces:full OR item_acces:mixed').narrow('digitized:T')
else:
sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value)))
return sqs
def myitems_index(request, view_name, page_title, no_items_message, index_name,
only_published=True, template="myitems/index.html"):
breadcrumbs = [
{"url": reverse("myitems:myitems"), "title": u"My Items"},
{"url": reverse("myitems:%s" % view_name), "title": page_title},
]
query_string_params = {}
index_params = IndexParams(request)
query_string_params = index_params.update_query_string_params(query_string_params)
batch_end = index_params.batch_start + index_params.batch_size
query = SearchQuerySet()
if only_published:
query = query.narrow("is_displayed:true")
query = query.narrow("%s:%i" % (index_name, request.user.id))
if index_params.query_order_by is not None:
query = query.order_by(index_params.query_order_by)
items = []
results = query[index_params.batch_start:batch_end]
for result in results:
items.append(populate_item_from_search_result(result))
pagination = Pagination(request.path, query_string_params,
index_params.batch_start,
index_params.batch_size,
len(query))
return direct_to_template(request, template, locals())
def search_api(request):
query = request.REQUEST.get("q", "")
start = int(request.REQUEST.get("start", 0))
limit = int(request.REQUEST.get("limit", getattr(settings, "HAYSTACK_SEARCH_RESULTS_PER_PAGE", 25)))
sort = request.REQUEST.get("sort", "relevance")
type = request.REQUEST.get("bytype")
sqs = SearchQuerySet()
if type is not None:
if type in ["map", "layer", "contact", "group"]:
# Type is one of our Major Types (not a sub type)
sqs = sqs.narrow("type:%s" % type)
elif type in ["vector", "raster"]:
# Type is one of our sub types
sqs = sqs.narrow("subtype:%s" % type)
if query:
sqs = sqs.filter(content=AutoQuery(query))
sqs = sqs.facet("type").facet("subtype")
if sort.lower() == "newest":
sqs = sqs.order_by("-date")
elif sort.lower() == "oldest":
sqs = sqs.order_by("date")
elif sort.lower() == "alphaaz":
sqs = sqs.order_by("title")
elif sort.lower() == "alphaza":
sqs = sqs.order_by("-title")
results = []
for i, result in enumerate(sqs[start:start + limit]):
data = json.loads(result.json)
data.update({"iid": i + start})
results.append(data)
facets = sqs.facet_counts()
counts = {"map": 0, "layer": 0, "vector": 0, "raster": 0, "contact": 0, "group": 0}
for t, c in facets.get("fields", {}).get("type", []):
counts[t] = c
for t, c in facets.get("fields", {}).get("subtype", []):
counts[t] = c
data = {
"success": True,
"total": sqs.count(),
"rows": results,
"counts": counts,
}
return HttpResponse(json.dumps(data), mimetype="application/json")
def search(self):
sqs = SearchQuerySet()
sqs = sqs.facet('bureaux').facet('section').facet('annee').facet(
'partenaire')
if self.q:
sqs = sqs.filter(content=sqs.query.clean(self.q))
# if self.courant:
# sqs = sqs.filter(date_fin__gte=datetime.date.today())
# if self.cloture:
# sqs = sqs.filter(date_fin__lt=datetime.date.today())
self.selected_facets = list(
set(self.selected_facets.split('&') + self.selected_facets_get))
for facet in self.selected_facets:
if "__" not in facet:
continue
field, value = facet.split("__", 1)
if value:
sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value)))
return sqs
def get_searchqueryset(self, form):
"""Get the Haystack searchqueryset (which we treat as
a regular Django queryset."""
sqs = SearchQuerySet()
if self.model:
sqs = sqs.models(self.model)
# FIXME: Move somewhere more sensible
if settings.PORTAL_HIDE_DRAFTS and not self.request.user.is_staff:
sqs = sqs.narrow("publication_status:%d" % models.Resource.PUBLISHED)
for facet in self.facetclasses:
sqs = facet.apply(sqs)
# apply the query
if form.is_valid():
sqs = form.filter(sqs)
for facetclass in self.facetclasses:
sqs = facetclass.narrow(sqs, self.request.GET.getlist(
facetclass.paramname))
counts = sqs.facet_counts()
current = sqs.query.narrow_queries
for facetclass in self.facetclasses:
facetclass.parse(counts, current)
# FIXME: Find way around assigning the sqs to the instance,
# but it seems to be difficult to prevent it from running
# multiple times otherwise, e.g. when checking for a spelling
# suggestion.
self.searchqueryset = sqs
return sqs
def get_items(self, from_date=None, until_date=None, set=None, microsite=None):
query = SearchQuerySet().narrow("is_displayed:true")
if from_date:
query = query.filter(published_on__gte=from_date)
if until_date:
query = query.filter(published_on__lte=until_date)
if set is not None:
set_name, slug = set.split(":")
if set_name == "collection":
query = query.narrow("collection:%i" % Collection.objects.get(slug=slug).id)
if microsite:
query = query.narrow("microsites:%i" % microsite.id)
return query.order_by("published_on").load_all()
def get_searchqueryset(self, form):
"""Get the Haystack searchqueryset (which we treat as
a regular Django queryset."""
sqs = SearchQuerySet()
if self.model:
sqs = sqs.models(self.model)
# FIXME: Move somewhere more sensible
if settings.PORTAL_HIDE_DRAFTS and not self.request.user.is_staff:
sqs = sqs.narrow("publication_status:%d" %
models.Resource.PUBLISHED)
for facet in self.facetclasses:
sqs = facet.apply(sqs)
# apply the query
if form.is_valid():
sqs = form.filter(sqs)
for facetclass in self.facetclasses:
sqs = facetclass.narrow(
sqs, self.request.GET.getlist(facetclass.paramname))
counts = sqs.facet_counts()
current = sqs.query.narrow_queries
for facetclass in self.facetclasses:
facetclass.parse(counts, current)
# FIXME: Find way around assigning the sqs to the instance,
# but it seems to be difficult to prevent it from running
# multiple times otherwise, e.g. when checking for a spelling
# suggestion.
self.searchqueryset = sqs
return sqs
def get_context(self, request):
context = super(RecordIndexPage, self).get_context(request)
# Get selected facets
selected_facets = set(request.GET.getlist('selected_facets'))
# Init a search query set
sqs = SearchQuerySet().models(RecordPage)
# Apply currently selected facets
for facet in selected_facets:
sqs = sqs.narrow(facet)
# Get facet counts
sqs = sqs.facet('language').facet('word_type').facet('first_letter')
# Generate presentable facet data
selected_facets_ui = []
for facet in selected_facets:
f = {
'value':
facet.split(':')[1],
'remove_url':
request.get_full_path().replace(
'&selected_facets={}'.format(facet), '')
}
selected_facets_ui.append(f)
context['selected_facets'] = selected_facets_ui
context['sqs'] = sqs
return context
def get_queryset(self, query_args):
# Create new queryset
qs = SearchQuerySet()
# Are we searching all models or just a specific one (depends on
# parameter set in View instantiation)
if self.search_model is not None:
qs = qs.models(self.search_model)
# Do we have a query or are we just getting all of them?
if 'q' in query_args:
qry = query_args['q']
## Currently deactivated due to policy decision to allow the users
## themselves to choose the granularity and fuzzyness of the search
##
# fuzzify search
# qry = u'{}~'.format(qry.replace(' ', '~ '))
qs = qs.auto_query(qry)
elif 'parl_id' in query_args:
qs = qs.filter(parl_id=query_args['parl_id'])
if 'llp_numeric' in query_args:
qs = qs.filter(llps_numeric=query_args['llp_numeric'])
return (qs.all(), None)
# Filter by facets
if query_args['facet_filters']:
for facet_field in query_args['facet_filters'].keys():
# We use narrow to limit the index entries beforehand, but
# need to use filter afterwards to remove partly correct results
# For instance, searching for Steyr (Oberoesterreich) yielded
# everyone from Oberoesterreich until filtering by it again.
qs = qs.narrow(u"{}:{}".format(
facet_field,
qs.query.clean(query_args['facet_filters'][facet_field])
)).filter(
**{facet_field: query_args['facet_filters'][facet_field]})
# Retrieve facets and facet_counts
facet_counts = []
if self.facet_fields:
facets = qs
for facet_field in self.facet_fields:
if self.facet_fields[facet_field]['type'] == 'date':
facets = facets.date_facet(
facet_field,
start_date=datetime.date(1900, 1, 1),
end_date=datetime.date(2050, 1, 1),
gap_by='month')
if self.facet_fields[facet_field]['type'] == 'field':
facets = facets.facet(facet_field)
facet_counts = facets.facet_counts()
# Get results and return them
if 'only_facets' in query_args:
result = {}
else:
result = qs.all()
return (result, facet_counts)
def get_queryset(cls, user):
queryset = SearchQuerySet()
queryset = queryset.models(*CREATED_MODELS)
queryset = queryset.narrow("is_displayed:true")
queryset = queryset.filter(SQ(creator=user.id))
return queryset
def search(self):
sqs = SearchQuerySet()
sqs = sqs.facet('bureaux').facet(
'section').facet('annee').facet('partenaire')
if self.q:
sqs = sqs.filter(content=sqs.query.clean(self.q))
if self.courant:
sqs = sqs.filter(date_fin__gte=datetime.date.today())
if self.cloture:
sqs = sqs.filter(date_fin__lt=datetime.date.today())
self.selected_facets = list(
set(self.selected_facets.split('&') + self.selected_facets_get))
for facet in self.selected_facets:
if "__" not in facet:
continue
field, value = facet.split("__", 1)
if value:
sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value)))
return sqs
def get_object(self, request):
self.queryDict = request.GET
all_results = SearchQuerySet().all()
facets = None
if 'selected_facets' in request.GET:
facets = request.GET.getlist('selected_facets')
if 'q' in request.GET:
self.query = request.GET['q']
results = all_results.filter(content=self.query)
if facets:
for facet in facets:
(facet_name, facet_value) = facet.split(':')
facet_name = facet_name.rsplit('_exact')[0]
results = results.narrow('%s:%s' %
(facet_name, facet_value))
elif facets:
for facet in facets:
(facet_name, facet_value) = facet.split(':')
facet_name = facet_name.rsplit('_exact')[0]
results = all_results.narrow('%s:%s' %
(facet_name, facet_value))
return results.order_by('-last_action_date')
def get_near_items(item):
#import ipdb; ipdb.set_trace()
qs = SearchQuerySet().all()
loc = item.locations.first()
point = Point(loc.lng, loc.lat)
qs = qs.dwithin('location', point, D(m=50000)).distance('location', point)
qs = qs.narrow('%s:"%s"' % ('groups_exact', 'public'))
return qs
def get_facets_for_field(field, model=None):
query = SearchQuerySet()
if model is not None:
query = query.models(model)
query = query.narrow("is_displayed:true")
query = query.facet(field)
return query.facet_counts().get("fields", {}).get(field, [])
def microsite(request, microsite):
microsite = get_object_or_404(Microsite, slug=microsite)
page_title = u"%s Home" % microsite.name
breadcrumbs = [{"url": reverse("materials:microsite", kwargs=dict(microsite=microsite.slug)), "title": page_title}]
query = SearchQuerySet().narrow("is_displayed:true")
query = query.narrow("microsites:%i" % microsite.id)
query = query.order_by("-rating")
query = query.facet("indexed_topics").facet("keywords").facet("grade_levels").facet("course_material_types")
items = []
results = query[0:8]
for result in results:
items.append(populate_item_from_search_result(result))
facets = query.facet_counts()["fields"]
topics = []
topic_counts = dict(facets["indexed_topics"])
for topic, tree_info in tree_item_iterator(microsite.topics.all()):
topic.count = topic_counts.get(str(topic.id), 0)
topics.append((topic, tree_info))
grade_levels = []
grade_level_counts = dict(facets["grade_levels"])
for level in GradeLevel.objects.all():
level.count = grade_level_counts.get(str(level.id), 0)
grade_levels.append(level)
course_material_types = []
course_material_type_counts = dict(facets["course_material_types"])
for material_type in CourseMaterialType.objects.all():
material_type.count = course_material_type_counts.get(str(material_type.id), 0)
course_material_types.append(material_type)
keywords = query.count() and facets.get("keywords", []) or []
if len(keywords) > MAX_TOP_KEYWORDS:
keywords = keywords[:MAX_TOP_KEYWORDS]
keywords = get_tag_cloud(dict(keywords), 3, 0, 0)
for keyword in keywords:
name = get_name_from_slug(Keyword, keyword["slug"]) or \
get_name_from_slug(Tag, keyword["slug"]) or \
keyword["slug"]
keyword["name"] = name
featured_k12 = SearchQuerySet().filter(workflow_state=PUBLISHED_STATE, featured=True, grade_levels__in=(1, 2), microsites=microsite.id).order_by("-featured_on").load_all()[:3]
featured_k12 = [r.object for r in featured_k12 if r]
featured_highered = SearchQuerySet().filter(workflow_state=PUBLISHED_STATE, featured=True, grade_levels=3, microsites=microsite.id).order_by("-featured_on").load_all()[:3]
featured_highered = [r.object for r in featured_highered if r]
slides = Slide.objects.filter(microsite=microsite)
resource_number = SearchQuerySet().filter(workflow_state=PUBLISHED_STATE, microsites=microsite.id).count()
return direct_to_template(request, "materials/microsites/%s.html" % microsite.slug, locals())
def get_queryset(self, query_args):
# Create new queryset
qs = SearchQuerySet()
# Are we searching all models or just a specific one (depends on
# parameter set in View instantiation)
if self.search_model is not None:
qs = qs.models(self.search_model)
# Do we have a query or are we just getting all of them?
if 'q' in query_args:
qry = query_args['q']
# fuzzify search
qry = u'{}~'.format(qry.replace(' ', '~ '))
qs = qs.auto_query(qry)
elif 'parl_id' in query_args:
qs = qs.filter(parl_id=query_args['parl_id'])
if 'llp_numeric' in query_args:
qs = qs.filter(llps_numeric=query_args['llp_numeric'])
return (qs.all(), None)
# Filter by facets
if query_args['facet_filters']:
for facet_field in query_args['facet_filters'].keys():
# We use narrow to limit the index entries beforehand, but
# need to use filter afterwards to remove partly correct results
# For instance, searching for Steyr (Oberoesterreich) yielded
# everyone from Oberoesterreich until filtering by it again.
qs = qs.narrow(u"{}:{}".format(
facet_field,
qs.query.clean(query_args['facet_filters'][facet_field]))
).filter(
**{
facet_field: query_args['facet_filters'][facet_field]}
)
# Retrieve facets and facet_counts
facet_counts = []
if self.facet_fields:
facets = qs
for facet_field in self.facet_fields:
if self.facet_fields[facet_field]['type'] == 'date':
facets = facets.date_facet(
facet_field,
start_date=datetime.date(1900, 1, 1),
end_date=datetime.date(2050, 1, 1),
gap_by='month')
if self.facet_fields[facet_field]['type'] == 'field':
facets = facets.facet(facet_field)
facet_counts = facets.facet_counts()
# Get results and return them
if 'only_facets' in query_args:
result = {}
else:
result = qs.all()
return (result, facet_counts)
def projectsheet_search(self, request, **kwargs):
""" Search project sheets. """
self.method_check(request, allowed=['get'])
self.throttle_check(request)
self.is_authenticated(request)
# Query params
query = request.GET.get('q', '')
autocomplete = request.GET.get('auto', None)
selected_facets = request.GET.getlist('facet', None)
order = request.GET.getlist('order', None)
sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags')
# narrow down QS with facets
if selected_facets:
for facet in selected_facets:
sqs = sqs.narrow('tags:%s' % (facet))
# A: if autocomplete, we return only a list of tags
# starting with "auto" along with their count.
if autocomplete is not None:
tags = sqs.facet_counts()
tags = tags['fields']['tags']
if len(autocomplete) > 0:
tags = [t for t in tags if t[0].startswith(autocomplete)]
tags = [{'name': t[0], 'count': t[1]} for t in tags]
object_list = {
'objects': tags,
}
# B: else, we return a list of projectsheets
else:
# launch query
if query != "":
sqs = sqs.auto_query(query)
uri = reverse('api_projectsheet_search',
kwargs={'api_name': self.api_name,
'resource_name': self._meta.resource_name})
paginator = Paginator(request.GET, sqs, resource_uri=uri)
objects = []
for result in paginator.page()['objects']:
if result:
bundle = self.build_bundle(obj=result.object,
request=request)
bundle = self.full_dehydrate(bundle, for_list=True)
objects.append(bundle)
object_list = {
'meta': paginator.page()['meta'],
'objects': objects,
}
self.log_throttled_access(request)
return self.create_response(request, object_list)
def search(self):
if not self.is_valid():
return self.no_query_found()
sqs = SearchQuerySet().models(Port).facet('maintainers').facet(
'categories').facet('variants')
# Performing narrowing based on facets
for facet in self.selected_facets:
if ":" not in facet:
continue
field, value = facet.split(":", 1)
if value:
sqs = sqs.narrow('%s:"%s"' % (field, sqs.query.clean(value)))
sort_by = "name_lower"
# Filter out deleted ports, based on query
if not self.cleaned_data.get('show_deleted_ports'):
sqs = sqs.filter(active=True)
# If a search query is present, only then perform the search operations
if self.cleaned_data.get('q'):
if self.cleaned_data['name']:
sqs = sqs.filter(name=self.cleaned_data['q'])
sort_by = "name_length"
else:
sqs = sqs.filter(
SQ(name=self.cleaned_data['q'])
| SQ(description=self.cleaned_data['q']))
sqs = sqs.order_by(sort_by)
# Filter operations, perform even if a search query is absent
# This is done to allow viewing all "outdated ports", "all ports with broken livecheck" etc.
f = SQ()
if self.cleaned_data['livecheck_uptodate']:
f = SQ(livecheck_broken=False) & SQ(livecheck_outdated=False)
if self.cleaned_data['livecheck_broken']:
f = f | SQ(livecheck_broken=True)
if self.cleaned_data['livecheck_outdated']:
f = f | SQ(livecheck_outdated=True)
if self.cleaned_data['nomaintainer']:
f = f & SQ(nomaintainer=True)
if self.cleaned_data['installed_file']:
f = f & SQ(files=Exact(self.cleaned_data['installed_file']))
if f != SQ():
sqs = sqs.filter(f)
return sqs
def green_browse(request):
microsite = get_object_or_404(Microsite, slug="green")
query = SearchQuerySet().narrow("is_displayed:true")
query = query.narrow("microsites:%i" % microsite.id)
query = query.facet("indexed_topics").facet("keywords").facet("grade_levels").facet("course_material_types")
facets = query.facet_counts()["fields"]
topics = []
topic_counts = dict(facets["indexed_topics"])
for topic, tree_info in tree_item_iterator(microsite.topics.all()):
topic.count = topic_counts.get(str(topic.id), 0)
topics.append((topic, tree_info))
grade_levels = []
grade_level_counts = dict(facets["grade_levels"])
for level in GradeLevel.objects.all():
level.count = grade_level_counts.get(str(level.id), 0)
grade_levels.append(level)
course_material_types = []
course_material_type_counts = dict(facets["course_material_types"])
for material_type in CourseMaterialType.objects.all():
material_type.count = course_material_type_counts.get(str(material_type.id), 0)
course_material_types.append(material_type)
keywords = query.count() and facets.get("keywords", []) or []
if len(keywords) > MAX_TOP_KEYWORDS:
keywords = keywords[:MAX_TOP_KEYWORDS]
keywords = get_tag_cloud(dict(keywords), 3, 0, 0)
for keyword in keywords:
name = get_name_from_slug(Keyword, keyword["slug"]) or \
get_name_from_slug(Tag, keyword["slug"]) or \
keyword["slug"]
keyword["name"] = name
query = SearchQuerySet().narrow("is_displayed:true")
query = query.narrow("microsites:%i" % microsite.id)
query = query.order_by("-published_on").load_all()
recently_added = [r.object for r in query[:7]]
return direct_to_template(request, "materials/microsites/green-browse.html", locals())
def file_search(self, request, **kwargs):
""" Implement file searching. """
self.method_check(request, allowed=['get'])
self.throttle_check(request)
self.is_authenticated(request)
# URL params
bucket_id = kwargs['bucket_id']
# Query params
query = request.GET.get('q', '')
autocomplete = request.GET.get('auto', None)
selected_facets = request.GET.getlist('facet', None)
order = request.GET.getlist('order', '-pub_date')
sqs = SearchQuerySet().models(BucketFile).filter(
bucket=bucket_id).order_by(order).facet('tags')
# 1st narrow down QS
if selected_facets:
for facet in selected_facets:
sqs = sqs.narrow('tags:%s' % (facet))
# A: if autocomplete, we return only a list of tags starting
# with "auto" along with their count.
if autocomplete is not None:
tags = sqs.facet_counts()
tags = tags['fields']['tags']
if len(autocomplete) > 0:
tags = [t for t in tags if t[0].startswith(autocomplete)]
tags = [
{'name': t[0], 'count':t[1]} for t in tags
]
object_list = {
'objects': tags,
}
# B: else, we return a list of files
else:
if query != "":
sqs = sqs.auto_query(query)
objects = []
# Building object list
for result in sqs:
bundle = self.build_bundle(obj=result.object, request=request)
bundle = self.full_dehydrate(bundle)
objects.append(bundle)
object_list = {
'objects': objects,
}
self.log_throttled_access(request)
return self.create_response(request, object_list)
def reindex_microsite_topic(topic):
from haystack.query import SearchQuerySet
objects = set()
# get all objects from this topic and all objects with this topic's keywords
query = "indexed_topics:%s" % topic.id
for result in SearchQuerySet().narrow(query).load_all():
objects.add(result.object)
topic_keywords = topic.keywords.values_list("slug", flat=True)
microsite_keywords = topic.microsite.keywords.values_list("slug", flat=True)
if topic_keywords and microsite_keywords:
query = SearchQuerySet()
query = query.narrow("keywords(%s)" % " OR ".join(["%s" % kw for kw in topic_keywords]))
query = query.narrow("keywords(%s)" % " OR ".join(["%s" % kw for kw in microsite_keywords]))
for result in query.load_all():
objects.add(result.object)
for instance in objects:
reindex(instance)
def ms_search(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
self.is_authenticated(request)
get_params = request.GET.copy()
if "advanced" in get_params:
return self.ms_advanced_search(request, **kwargs)
# Query params
query = get_params.get('q', None)
selected_facets = [
item for sublist in
[f.split(',') for f in get_params.getlist('facet', "")]
for item in sublist
]
selected_facets = None if selected_facets == [''] else selected_facets
ordering = get_params.get('ordering', None)
limit = get_params.get('limit', self._meta.limit)
for word in ["q", "facet", "ordering", "format", 'limit', 'offset']:
if word in get_params.keys():
try:
del get_params[word]
except KeyError:
print word, "not in request"
filtering = {}
for key, val in get_params.iteritems():
filtering[key] = json.loads(val)
sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags')
if selected_facets:
first_narrow_succes = False
for i, facet in enumerate(selected_facets):
tmp_sqs = sqs.narrow('tags_exact:%s' % slugify(facet))
if len(tmp_sqs):
first_narrow_succes = True
if first_narrow_succes or i == len(selected_facets) - 1:
sqs = tmp_sqs
if query:
sqs = sqs.filter(text=Raw(query))
if filtering:
sqs = sqs.filter(**filtering)
if ordering:
sqs = sqs.order_by(ordering)
self.log_throttled_access(request)
return self.create_response(request,
self.prepare_result(request, sqs, limit))
def search(self):
kwargs = {
"hl.simple.pre": "<mark>",
"hl.simple.post": "</mark>",
"hl.fragsize": 400,
"hl.snippets": 3,
}
sqs = self.no_query_found()
# use the text_exact field in solr which doesn't stem
if self.cleaned_data["q"] and self.cleaned_data["exact"] is True:
query = self.cleaned_data["q"]
kwargs["hl.fl"] = "text_exact"
sqs = SearchQuerySet().filter(text_exact=Exact(query))
if self.load_all:
sqs = sqs.load_all()
else:
sqs = super(HighlightedSearchForm, self).search()
if self.cleaned_data["similar_type"]:
similar_council = self.cleaned_data["similar_council"]
comparison_type = self.cleaned_data["similar_type"]
cut_off = 15
names = Distance.objects.filter(
council_a=similar_council,
type=comparison_type,
position__lte=cut_off,
).values_list("council_b__name", flat=True)
query_names = [f'council_name:"{name}"' for name in names]
sqs = sqs.narrow(" OR ".join(query_names))
if self.cleaned_data["council_name"]:
# narrow makes use of fq rather than q
sqs = sqs.narrow("council_name:%s" %
self.cleaned_data["council_name"])
return sqs.highlight(**kwargs)
def results(request):
url = urllib.unquote_plus(request.get_full_path())
print url
searchform = SearchForm(request.GET)
if not "q" in request.GET:
return redirect('search.views.index')
if searchform.is_valid():
q = searchform.cleaned_data['q']
selected_facets = request.GET.getlist("selected_facets")
remove_facets = request.GET.getlist("remove_facets")
applied_facets = {}
if remove_facets:
for r in remove_facets:
filter(lambda a: a != r, selected_facets)
filter(lambda a: a != r, applied_facets)
if selected_facets:
for f in selected_facets:
k, v = f.split(":")
myurl = url.replace("&selected_facets=%s" % f, "")
tag = "%s : %s" % (k.upper(), v)
applied_facets[tag] = myurl
print applied_facets
if q:
sqs = SearchQuerySet().models(Part).models(Company).facet('brand').facet('category').facet('with_distributors').facet('with_image').auto_query(q)
for facet in selected_facets:
sqs = sqs.narrow(facet)
try:
page = request.GET.get('page', 1)
except PageNotAnInteger:
page = 1
p = Paginator(sqs, 10, request=request)
results_list = p.page(page)
return render_to_response('search/results.html',
{
'results_list': results_list,
'query': q,
'facets': sqs.facet_counts(),
'applied_facets': applied_facets,
},
context_instance=RequestContext(request))
def ms_search(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
self.is_authenticated(request)
get_params = request.GET.copy()
if "advanced" in get_params:
return self.ms_advanced_search(request, **kwargs)
# Query params
query = get_params.get('q', None)
selected_facets = [item for sublist in [f.split(',') for f in get_params.getlist('facet', "")] for item in sublist]
selected_facets = None if selected_facets == [''] else selected_facets
ordering = get_params.get('ordering', None)
limit = get_params.get('limit', self._meta.limit)
for word in ["q", "facet", "ordering", "format", 'limit', 'offset']:
if word in get_params.keys():
try:
del get_params[word]
except KeyError:
print word, "not in request"
filtering = {}
for key, val in get_params.iteritems():
filtering[key] = json.loads(val)
sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags')
if selected_facets:
first_narrow_succes = False
for i, facet in enumerate(selected_facets):
tmp_sqs = sqs.narrow('tags_exact:%s' % slugify(facet))
if len(tmp_sqs) :
first_narrow_succes = True
if first_narrow_succes or i == len(selected_facets) - 1:
sqs = tmp_sqs
if query:
sqs = sqs.filter(text=Raw(query))
if filtering :
sqs = sqs.filter(**filtering)
if ordering:
sqs = sqs.order_by(ordering)
self.log_throttled_access(request)
return self.create_response(request, self.prepare_result(request, sqs, limit))
def get_facet_counts(self):
queryset = SearchQuerySet().models(PostAssertion)
queryset = self._apply_facets_to_queryset(queryset)
params = self.request.GET
if ('date_from' in params and params['date_from']) or \
('date_to' in params and params['date_to']):
query = 'date:[{} TO {}]'.format(
-1 * int(params['date_from'])
if 'date_from' in params and params['date_from']
else PromrepFacetedSearchForm.MIN_DATE,
-1 * int(params['date_to'])
if 'date_to' in params and params['date_to']
else PromrepFacetedSearchForm.MAX_DATE
)
queryset = queryset.narrow(query)
for field in PromrepFacetedSearchForm.AUTOCOMPLETE_FACETS:
if field in params and params[field]:
queryset = queryset.narrow(
'{}:{}'.format(field, params[field]))
return queryset.facet_counts()
def get_facets_for_field(field, model=None, facet_limit=None, facet_mincount=None):
query = SearchQuerySet()
if model is not None:
if isinstance(model, (tuple, list)):
query = query.models(*model)
else:
query = query.models(model)
query = query.narrow("is_displayed:true")
query = query.facet(field)
if facet_limit:
query = query.facet_limit(facet_limit)
if facet_mincount:
query = query.facet_mincount(facet_mincount)
return query.facet_counts().get("fields", {}).get(field, [])
def featured_counts(obj):
try:
featured_counts = {}
sqs = SearchQuerySet()
sqs = sqs.facet("featured")
sqs = sqs.narrow("featured:[* TO *]")
for featured in sqs.facet_counts()['fields']['featured']:
key = featured[0].replace(' ', '')
key = key.replace('-', '')
featured_counts[key] = featured[1]
return featured_counts
except:
return None
def featured_counts(obj):
try:
featured_counts = {}
sqs = SearchQuerySet()
sqs = sqs.facet("featured")
sqs = sqs.narrow("featured:[* TO *]")
for featured in sqs.facet_counts()['fields']['featured']:
key = featured[0].replace(' ', '')
key = key.replace('-', '')
featured_counts[key] = featured[1]
return featured_counts
except KeyError:
return None
def get_queryset(self):
queryset = super(SenateSearchView, self).get_queryset()
if 'senate_date' in self.request.GET:
queryset = SearchQuerySet().models(StatusAssertion)
else:
queryset = queryset.narrow('date:[{0} TO {0}]'.format(
SenateSearchForm.INITIAL_DATE))
certainty = self.request.GET.get('dating_certainty')
if certainty is not None and certainty == '3':
return queryset.order_by('-date_end')
return queryset.order_by('date_start')
def facet_counts(obj, facetName):
try:
counts = {}
sqs = SearchQuerySet()
sqs = sqs.facet(facetName)
sqs = sqs.narrow(facetName + ":[* TO *]")
for x in sqs.facet_counts()['fields'][facetName]:
key = x[0].replace(' ', '')
key = key.replace('-', '')
counts[key] = x[1]
return counts
except:
return None
def infrastructure_counts(obj):
try:
infrastructure_counts = {}
sqs = SearchQuerySet()
sqs = sqs.facet("infrastructure")
sqs = sqs.narrow("infrastructure:[* TO *]")
for infrastructure in sqs.facet_counts()['fields']['infrastructure']:
key = infrastructure[0].replace(' ', '')
key = key.replace('-', '')
infrastructure_counts[key] = infrastructure[1]
return infrastructure_counts
except:
return None
def ask_me_anything(request):
context = {}
if request.method == 'POST':
q = request.POST.get('q', '')
if q:
q = Question(q)
sq = SearchQuerySet()
if q.get_type() in ('LOC', 'DATETIME', 'NUM', 'PERSON'):
sq = sq.narrow('ann:' + q.get_type())
for k in q.weight_keywords():
sq = sq.filter_or(SQ(content=k) | SQ(title=k) | SQ(context=k))
context = {
'q': q,
'result': sq[:10],
}
return render(request, 'ask_me_anything.html', context)
def infrastructure_counts(obj):
try:
infrastructure_counts = {}
sqs = SearchQuerySet()
sqs = sqs.facet("infrastructure")
sqs = sqs.narrow("infrastructure:[* TO *]")
for infrastructure in sqs.facet_counts()['fields']['infrastructure']:
key = infrastructure[0].replace(' ', '')
key = key.replace('-', '')
infrastructure_counts[key] = infrastructure[1]
return infrastructure_counts
except KeyError:
return None
def facet_counts(obj, facetName):
try:
counts = {}
sqs = SearchQuerySet()
sqs = sqs.facet(facetName)
sqs = sqs.narrow(facetName + ":[* TO *]")
for x in sqs.facet_counts()['fields'][facetName]:
key = x[0].replace(' ', '')
key = key.replace('-', '')
counts[key] = x[1]
return counts
except KeyError:
return None
def bulk_payment_ajax_search(request):
sqs = SearchQuerySet()
sqs = sqs.narrow('content_type_exact:"Bulk Payment"')
if 'q' in request.GET:
q = request.GET['q']
sqs = sqs.filter(**{'mixed_name__startswith': q})
return [{
'text': "(#%d) %s %d%s %s" % (
bulk_payment.object.pk,
bulk_payment.object.person.mixed_name,
bulk_payment.amount,
bulk_payment.currency,
bulk_payment.receipt_date.strftime("(%B %-d %Y)"),
),
'value': bulk_payment.object.pk
} for bulk_payment in sqs]
def list(self):
from haystack.query import SearchQuerySet
results = SearchQuerySet()
for facet in Facet.asset_facets:
results = results.facet(facet)
types = self.search_criteria.split(';')
for t in types:
criteria = t.split(',')
q = ''
for c in criteria:
if len(q):
q += ' OR '
q += c.strip()
results = results.narrow(q)
return results.order_by("name")
def list(self):
from haystack.query import SearchQuerySet
results = SearchQuerySet()
for facet in Facet.asset_facets:
results = results.facet(facet)
types = self.search_criteria.split(';')
for t in types:
criteria = t.split(',')
q = ''
for c in criteria:
if len(q):
q += ' OR '
q += c.strip()
results = results.narrow(q)
return results.order_by("name")
def get_queryset(self):
queryset = SearchQuerySet().filter(active=True).order_by('first_name')
# further filter queryset based on some set of criteria
party = self.request.GET.getlist('party', '')
party_or = ""
college = self.request.GET.getlist('college', '')
college_or = ""
state = self.request.GET.getlist('state', '')
state_or = ""
issues = self.request.GET.getlist('issues', '')
issues_or = ""
race = self.request.GET.getlist('race', '')
race_or = ""
race_type = self.request.GET.getlist('race_type', '')
race_type_or = ""
women = self.request.GET.get('women', '')
q = self.request.GET.get('q', '')
page = self.request.GET.get('page', '')
search = ''.join(party) + q + page
if search == "":
search = "null"
if party:
for facet in party:
party_or += 'party: "%s"' % (facet)
queryset = queryset.narrow(party_or)
if college:
for facet in college:
college_or += 'college: "%s"' % (facet)
queryset = queryset.narrow(college_or)
if state:
for facet in state:
state_or += 'state: "%s"' % (facet)
queryset = queryset.narrow(state_or)
if issues:
for facet in issues:
issues_or += 'issues: "%s"' % (facet)
queryset = queryset.narrow(issues_or)
if race:
for facet in race:
race_or += 'race: "%s"' % (facet)
queryset = queryset.narrow(race_or)
if race_type:
for facet in race_type:
race_type_or += 'race_type: "%s"' % (facet)
queryset = queryset.narrow(race_type_or)
if q:
queryset = queryset.filter(
SQ(text=AutoQuery(q)) | SQ(title=AutoQuery(q)))
if women:
queryset = queryset.filter(women=False)
return queryset
def projectsheet_search(self, request, **kwargs):
self.method_check(request, allowed=['get'])
self.throttle_check(request)
self.is_authenticated(request)
# Query params
query = request.GET.get('q', '')
selected_facets = request.GET.getlist('facet', None)
sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags')
# narrow down QS with facets
if selected_facets:
for facet in selected_facets:
sqs = sqs.narrow('tags:%s' % (facet))
# launch query
if query != "":
sqs = sqs.auto_query(query)
uri = reverse('api_projectsheet_search',
kwargs={'api_name': self.api_name,
'resource_name': self._meta.resource_name})
paginator = Paginator(request.GET, sqs, resource_uri=uri)
objects = []
for result in paginator.page()['objects']:
if result:
bundle = self.build_bundle(obj=result.object, request=request)
bundle = self.full_dehydrate(bundle)
objects.append(bundle)
object_list = {
'meta': paginator.page()['meta'],
'objects': objects,
}
self.log_throttled_access(request)
return self.create_response(request, object_list)
def get_queryset(self, query_args):
# Create new queryset
qs = SearchQuerySet()
# Are we searching all models or just a specific one (depends on
# parameter set in View instantiation)
if self.search_model is not None:
qs = qs.models(self.search_model)
# Do we have a query or are we just getting all of them?
if 'q' in query_args:
qs = qs.auto_query(query_args['q'])
# Filter by facets
if query_args['facet_filters']:
# qs.filter(**query_args['facet_filters'])
for facet_field in query_args['facet_filters'].keys():
qs = qs.narrow(u"{}:{}".format(
facet_field,
query_args['facet_filters'][facet_field])
)
# Retrieve facets and facet_counts
facet_counts = []
if self.facet_fields:
facets = qs
for facet_field in self.facet_fields:
facets = facets.facet(facet_field)
facet_counts = facets.facet_counts()
# Get results and return them
if 'only_facets' in query_args:
result = {}
else:
result = qs.all()
return (result, facet_counts)
def all_notes(request, project_slug=None):
o = {}
template = 'all-notes.html'
o['filtered'] = False
if request.GET.get('filter'):
template = 'filtered-notes.html'
o['filtered'] = True
qs = SearchQuerySet().models(Note)
query = []
if request.GET.get('topic'):
query += [ ' AND '.join([ 'related_topic_id:%s' % topic for topic
in request.GET.get('topic').split(',') ]) ]
if request.GET.get('project'):
query += [ ' AND '.join([ 'project_id:%s' % project for project
in request.GET.get('project').split(',') ]) ]
qs = qs.narrow(' AND '.join(query)) if query else qs
qs = qs.facet('related_topic_id').facet('project_id')
facet_fields = qs.facet_counts()['fields']
topic_facets = sorted(facet_fields['related_topic_id'],
key=lambda t: t[1], reverse=True)
project_facets = sorted(facet_fields['project_id'],
key=lambda p: p[1], reverse=True)
topic_facets = [ (Topic.objects.get(id=t_id), t_count)
for t_id, t_count in topic_facets[:16] ]
o['topic_facets_1'] = topic_facets[:8]
o['topic_facets_2'] = topic_facets[8:] if (len(topic_facets) > 8) else []
o['project_facets'] = [ (Project.objects.get(id=p_id), p_count)
for p_id, p_count in project_facets ]
o['notes'] = qs
return render_to_response(
template, o, context_instance=RequestContext(request))
def get_object(self, request):
self.queryDict = request.GET
all_results = SearchQuerySet().all()
facets = None
if 'selected_facets' in request.GET:
facets = request.GET.getlist('selected_facets')
if 'q' in request.GET:
self.query = request.GET['q']
results = all_results.filter(content=self.query)
if facets:
for facet in facets:
(facet_name, facet_value) = facet.split(':')
facet_name = facet_name.rsplit('_exact')[0]
results = results.narrow('%s:%s' % (facet_name, facet_value))
elif facets:
for facet in facets:
(facet_name, facet_value) = facet.split(':')
facet_name = facet_name.rsplit('_exact')[0]
results = all_results.narrow('%s:%s' % (facet_name, facet_value))
return results.order_by('-last_action_date')
def infrastructure_assets(obj):
# Get all assets with valid "infrastructure" facets
sqs = SearchQuerySet()
sqs = sqs.facet("infrastructure")
sqs = sqs.narrow("infrastructure:[* TO *]")
return sqs
def facet_assets(obj, facetName):
# Get all assets with valid "infrastructure" facets
sqs = SearchQuerySet()
sqs = sqs.facet(facetName)
sqs = sqs.narrow(facetName + ":[* TO *]")
return sqs
def get_context(self, request, *args, **kwargs):
facet_map = (
('building_block', (ActivityBuildingBlock, False, 10)),
('school_subject', (ActivitySchoolSubject, False, 25)),
('topic', (ActivityTopic, True, 25)),
('grade_level', (ActivityGradeLevel, False, 10)),
('age_range', (ActivityAgeRange, False, 10)),
('student_characteristics', (ActivityStudentCharacteristics, False,
10)), # noqa: E501
('activity_type', (ActivityType, False, 10)),
('teaching_strategy', (ActivityTeachingStrategy, False, 25)),
('blooms_taxonomy_level', (ActivityBloomsTaxonomyLevel, False,
25)), # noqa: E501
('activity_duration', (ActivityDuration, False, 10)),
('jump_start_coalition', (ActivityJumpStartCoalition, False, 25)),
('council_for_economic_education', (ActivityCouncilForEconEd,
False, 25)), # noqa: E501
)
search_query = request.GET.get('q', '') # haystack cleans this string
sqs = SearchQuerySet().models(ActivityPage).filter(live=True)
total_activities = sqs.count()
# Load selected facets
selected_facets = {}
facet_queries = {}
for facet, facet_config in facet_map:
sqs = sqs.facet(str(facet), size=facet_config[2])
if facet in request.GET and request.GET.get(facet):
selected_facets[facet] = [
int(value) for value in request.GET.getlist(facet)
if value.isdigit()
]
facet_queries[facet] = facet + '_exact:' + (
" OR " + facet + "_exact:").join(
[str(value) for value in selected_facets[facet]])
payload = {
'search_query': search_query,
'results': [],
'total_results': 0,
'total_activities': total_activities,
'selected_facets': selected_facets,
'facet_queries': facet_queries,
'all_facets': {},
}
# Apply search query if it exists, but don't apply facets
if search_query:
sqs = sqs.filter(content=search_query).order_by(
'-_score', '-date') # noqa: E501
else:
sqs = sqs.order_by('-date')
# Get all facets and their counts
facet_counts = sqs.facet_counts()
all_facets = self.get_all_facets(facet_map, sqs, facet_counts,
facet_queries,
selected_facets) # noqa: E501
# List all facet blocks that need to be expanded
always_expanded = {'building_block', 'topic', 'school_subject'}
conditionally_expanded = {
facet_name
for facet_name, facet_items in all_facets.items()
if any(facet['selected'] is True for facet in facet_items)
}
expanded_facets = always_expanded.union(set(conditionally_expanded))
payload.update({
'facet_counts': facet_counts,
'all_facets': all_facets,
'expanded_facets': expanded_facets,
})
# Apply all the active facet values to our search results
for facet_narrow_query in facet_queries.values():
sqs = sqs.narrow(facet_narrow_query)
results = [activity.object for activity in sqs]
total_results = sqs.count()
payload.update({
'results': results,
'total_results': total_results,
})
self.results = payload
results_per_page = validate_results_per_page(request)
paginator = Paginator(payload['results'], results_per_page)
current_page = validate_page_number(request, paginator)
paginated_page = paginator.page(current_page)
context = super(ActivityIndexPage, self).get_context(request)
context.update({
'facet_counts': facet_counts,
'facets': all_facets,
'activities': paginated_page,
'total_results': total_results,
'results_per_page': results_per_page,
'current_page': current_page,
'paginator': paginator,
'show_filters': bool(facet_queries),
})
return context
class SearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections['default']._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections['default']._index = self.ui
# Update the "index".
backend = connections['default'].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections['default']._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r'^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object'
r' at 0x[0-9A-Fa-f]+>, using=None>$')
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in msqs]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(check, [
u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'10', u'11',
u'12', u'13', u'14', u'15', u'16', u'17', u'18', u'19', u'20',
u'21', u'22', u'23'
])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections['default']._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [u'sometext', u'1234'])
self.assertEqual(len(connections['default'].queries), 1)
connections['default']._index = old_ui
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
sqs = SearchQuerySet().all()
sqs.query.backend = MixedMockSearchBackend('default')
results = sqs
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 0)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [])
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 9)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10])
self.assertEqual(len(connections['default'].queries), 2)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 17)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20])
self.assertEqual(len(connections['default'].queries), 4)
results._fill_cache(20, 30)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [
1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22,
23
])
self.assertEqual(len(connections['default'].queries), 6)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue('foo' in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections['default']._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections['default']._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost('foo', 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content='Indx')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search('foo')), 23)
self.assertEqual(
len(
self.msqs.raw_search(
'(content__exact:hello AND content__exact:world)')), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend('charpk')
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# If nothing is handled, you get nothing.
old_ui = connections['default']._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections['default']._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections['default']._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections['default']._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections['default']._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# Restore.
connections['default']._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query('test search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains=test search -stuff>')
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search -stuff>')
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search \'moar quotes\' -stuff>'
)
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search \'moar quotes\' "foo -stuff>'
)
sqs = self.msqs.auto_query('test - stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND content__contains=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query('test search -stuff', fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND title__contains=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff',
fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND title__contains=test "my thing" search -stuff>')
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet('foo').facet('bar')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='smarblaph')
self.fail()
except FacetingError as e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second."
)
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month').date_facet(
'bar',
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='year')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet('foo', '[bar TO *]')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet('foo', '[bar TO *]').query_facet(
'bar', '[100 TO 499]')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = self.msqs.query_facet('foo', '[bar TO *]').query_facet(
'bar', '[100 TO 499]').query_facet('foo', '[1000 TO 1499]')
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet('foo', 'bar')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet('foo', 'bar').stats_facet('foo', 'baz')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet('foo', 'bar').stats_facet('moof', 'baz')
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow('foo:moof')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo='bar', foo__lt='10')
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using='default')
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, 'default')
def test_chaining(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content='bar')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content='foo')
sqs2 = self.msqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content='foo')
sqs2 = self.msqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content='foo').filter(content='oof')
sqs2 = self.msqs.filter(content='bar').filter(content='rab')
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, 'OR')
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content='foo').filter_or(content='oof')
sqs2 = self.msqs.filter(content='bar').filter_or(content='rab')
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, 'AND')
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def haystack_search_api(request, format="json", **kwargs):
"""
View that drives the search api
"""
from haystack.inputs import Raw
from haystack.query import SearchQuerySet, SQ
# Retrieve Query Params
id = request.REQUEST.get("id", None)
query = request.REQUEST.get('q', None)
category = request.REQUEST.get("category", None)
limit = int(
request.REQUEST.get(
"limit", getattr(settings, "HAYSTACK_SEARCH_RESULTS_PER_PAGE",
20)))
startIndex = int(request.REQUEST.get("startIndex", 0))
sort = request.REQUEST.get("sort", "relevance")
type_facets = request.REQUEST.get("type", None)
format = request.REQUEST.get("format", format)
date_start = request.REQUEST.get("start_date", None)
date_end = request.REQUEST.get("end_date", None)
keyword = request.REQUEST.get("kw", None)
service = request.REQUEST.get("service", None)
local = request.REQUEST.get("local", None)
# Geospatial Elements
bbox = request.REQUEST.get("extent", None)
ts = time()
sqs = SearchQuerySet()
limit = min(limit, 500)
# Filter by ID
if id:
sqs = sqs.narrow("django_id:%s" % id)
# Filter by Type and subtype
if type_facets is not None:
type_facets = type_facets.replace("owner", "user").split(",")
subtype_facets = ["vector", "raster"]
types = []
subtypes = []
for type in type_facets:
if type in ["map", "layer", "user", "document", "group"]:
# Type is one of our Major Types (not a sub type)
types.append(type)
elif type in subtype_facets:
subtypes.append(type)
if len(subtypes) > 0:
for sub_type in subtype_facets:
if sub_type not in subtypes:
sqs = sqs.exclude(subtype='%s' % sub_type)
if len(types) > 0:
sqs = sqs.narrow("type:%s" % ','.join(map(str, types)))
# Filter by Query Params
# haystack bug? if boosted fields aren't included in the
# query, then the score won't be affected by the boost
if query:
if query.startswith('"') or query.startswith('\''):
#Match exact phrase
phrase = query.replace('"', '')
sqs = sqs.filter(
SQ(title__exact=phrase) | SQ(abstract__exact=phrase)
| SQ(content__exact=phrase))
else:
words = query.split()
for word in range(0, len(words)):
if word == 0:
sqs = sqs.filter(
SQ(title=Raw(words[word]))
| SQ(abstract=Raw(words[word]))
| SQ(content=Raw(words[word])))
elif words[word] in ["AND", "OR"]:
pass
elif words[word - 1] == "OR": #previous word OR this word
sqs = sqs.filter_or(
SQ(title=Raw(words[word]))
| SQ(abstract=Raw(words[word]))
| SQ(content=Raw(words[word])))
else: #previous word AND this word
sqs = sqs.filter(
SQ(title=Raw(words[word]))
| SQ(abstract=Raw(words[word]))
| SQ(content=Raw(words[word])))
# filter by cateory
if category:
sqs = sqs.narrow('category:%s' % category)
#filter by keyword
if keyword:
sqs = sqs.narrow('keywords:%s' % keyword)
if date_start:
sqs = sqs.filter(SQ(date__gte=date_start))
if date_end:
sqs = sqs.filter(SQ(date__lte=date_end))
"""
### Code to filter on temporal extent start/end dates instead
if date_start or date_end:
#Exclude results with no dates at all
sqs = sqs.filter(
SQ(temporal_extent_start=Raw("[* TO *]")) | SQ(temporal_extent_end=Raw("[* TO *]"))
)
if temporal_start and temporal_end:
#Return anything with a start date < date_end or an end date > date_start
sqs = sqs.filter(
SQ(temporal_extent_end__gte=date_start) | SQ(temporal_extent_start__lte=date_end)
)
elif temporal_start:
#Exclude anything with an end date <date_start
sqs = sqs.exclude(
SQ(temporal_extent_end__lte=date_start)
)
elif temporal_end:
#Exclude anything with a start date > date_end
sqs = sqs.exclude(
SQ(temporal_extent_start__gte=date_end)
)
"""
if bbox:
left, right, bottom, top = bbox.split(',')
sqs = sqs.filter(
# first check if the bbox has at least one point inside the window
SQ(bbox_left__gte=left) & SQ(bbox_left__lte=right)
& SQ(bbox_top__gte=bottom) & SQ(bbox_top__lte=top)
| #check top_left is inside the window
SQ(bbox_right__lte=right) & SQ(bbox_right__gte=left)
& SQ(bbox_top__lte=top) & SQ(bbox_top__gte=bottom)
| #check top_right is inside the window
SQ(bbox_bottom__gte=bottom) & SQ(bbox_bottom__lte=top)
& SQ(bbox_right__lte=right) & SQ(bbox_right__gte=left)
| #check bottom_right is inside the window
SQ(bbox_top__lte=top) & SQ(bbox_top__gte=bottom)
& SQ(bbox_left__gte=left) & SQ(bbox_left__lte=right)
| #check bottom_left is inside the window
# then check if the bbox is including the window
SQ(bbox_left__lte=left) & SQ(bbox_right__gte=right)
& SQ(bbox_bottom__lte=bottom) & SQ(bbox_top__gte=top))
# Filter by permissions
'''
### Takes too long with many results.
### Instead, show all results but disable links on restricted ones.
for i, result in enumerate(sqs):
if result.type == 'layer':
if not request.user.has_perm('layers.view_layer',obj = result.object):
sqs = sqs.exclude(id = result.id)
if result.type == 'map':
if not request.user.has_perm('maps.view_map',obj = result.object):
sqs = sqs.exclude(id = result.id)
'''
#filter by service
'''
if service:
sqs = sqs.narrow('service:%s' % service)
if local:
sqs = sqs.narrow('local:%s' % local)
'''
# Apply Sort
# TODO: Handle for Revised sort types
# [relevance, alphabetically, rating, created, updated, popularity]
if sort.lower() == "newest":
sqs = sqs.order_by("-modified")
elif sort.lower() == "oldest":
sqs = sqs.order_by("modified")
elif sort.lower() == "alphaaz":
sqs = sqs.order_by("title_sortable")
elif sort.lower() == "alphaza":
sqs = sqs.order_by("-title_sortable")
elif sort.lower() == "popularity":
sqs = sqs.order_by("-popular_count")
else:
sqs = sqs.order_by("-_score")
# Setup Search Results
results = []
items = []
# Build the result based on the limit
for i, result in enumerate(sqs[startIndex:startIndex + limit]):
logger.info(result)
data = result.get_stored_fields()
resource = None
if "modified" in data:
data["modified"] = data["modified"].strftime(
"%Y-%m-%dT%H:%M:%S.%f")
if "temporal_extent_start" in data and data[
"temporal_extent_start"] is not None:
data["temporal_extent_start"] = data[
"temporal_extent_start"].strftime("%Y-%m-%dT%H:%M:%S.%f")
if "temporal_extent_end" in data and data[
"temporal_extent_end"] is not None:
data["temporal_extent_end"] = data["temporal_extent_end"].strftime(
"%Y-%m-%dT%H:%M:%S.%f")
if data['type'] == "map":
resource = MapNormalizer(Map.objects.get(pk=data['oid']))
elif data['type'] == "layer":
resource = LayerNormalizer(Layer.objects.get(pk=data['oid']))
elif data['type'] == "user":
resource = OwnerNormalizer(Profile.objects.get(pk=data['oid']))
elif data['type'] == "document":
resource = DocumentNormalizer(Document.objects.get(pk=data['oid']))
elif data[
'type'] == "group" and "geonode.contrib.groups" in settings.INSTALLED_APPS:
resource = GroupNormalizer(Group.objects.get(pk=data['oid']))
if resource:
resource.rating = data["rating"] if "rating" in data else 0
results.append(data)
items.append(resource)
# Setup Facet Counts
sqs = sqs.facet("type").facet("subtype")
sqs = sqs.facet('category')
sqs = sqs.facet('keywords')
sqs = sqs.facet('service')
sqs = sqs.facet('local')
facet_counts = sqs.facet_counts()
# Prepare Search Results
data = {
"success":
True,
"total":
sqs.count(),
"query_info": {
"q": query,
"startIndex": startIndex,
"limit": limit,
"sort": sort,
"type": type_facets,
},
"results":
results,
"facets":
dict(
facet_counts.get("fields")['type'] +
facet_counts.get('fields')['subtype']) if sqs.count() > 0 else [],
"categories": {
facet[0]: facet[1]
for facet in facet_counts.get('fields')['category']
} if sqs.count() > 0 else {},
"keywords": {
facet[0]: facet[1]
for facet in facet_counts.get('fields')['keywords']
} if sqs.count() > 0 else {},
}
# Return Results
ts1 = time() - ts
if format == "html": #Send to search/explore page
return data, items
elif format == "raw":
return HttpResponse(json.dumps(data), mimetype="application/json")
else:
query = query_from_request(request, kwargs)
return _search_json(query, items, data["facets"], ts1)
def get_queryset(self, *args, **kwargs):
qs = SearchQuerySet().all()
qs = qs.facet('categories')
if self.request.user.is_authenticated():
qs = qs.facet('groups')
if self.request.query_params.get('q'):
#qs = qs.filter_and(title__contains=self.request.query_params.get('q'))
qs = qs.auto_query(self.request.query_params.get('q'))
distance = None
try:
for k, v in self.request.query_params.items():
if k in D.UNITS.keys():
distance = {k: v}
except Exception as e:
logging.error(e)
point = None
try:
point = Point(float(self.request.query_params['lng']),
float(self.request.query_params['lat']))
except Exception as e:
logging.error(e)
if distance and point:
qs = qs or SearchQuerySet()
qs = qs.dwithin('location', point,
D(**distance)).distance('location', point)
try:
if self.request.query_params.get('w') and self.request.query_params.get('s')\
and self.request.query_params.get('n') and self.request.query_params.get('e'):
bottom_left = Point(float(self.request.query_params['w']),
float(self.request.query_params['s']))
top_right = Point(float(self.request.query_params['e']),
float(self.request.query_params['n']))
qs = qs.within('location', bottom_left, top_right)
except:
pass
param_facet_url = self.request.query_params.get('selected_facets',
'').split(',')
flag_group = False
for param_facet in param_facet_url:
if ":" not in param_facet:
continue
field, value = param_facet.split(":", 1)
if value and field.split('_')[1] == 'exact':
try:
# Validate that the user belongs to the group
if field.split('_')[0] == 'groups':
InterestGroup.objects.\
filter(Q(memberships__user=self.request.user) | Q(owner=self.request.user)).\
get(slug=value)
flag_group = True
qs = qs.narrow('%s:"%s"' % (field, qs.query.clean(value)))
except InterestGroup.DoesNotExist:
# The user hasnt permissiont to filter by this group.
pass
self.facets = []
group_facet = get_facet_by_name(param_facet_url,
qs.facet_counts()['fields'].items(),
'groups', self.request.user)
if group_facet:
self.facets.append(group_facet)
order_by = self.request.query_params.get('order_by')
if order_by:
# TODO: Order by distance not found
if order_by is 'distance':
if distance and point:
qs = qs.order_by(order_by)
else:
qs = qs.order_by(order_by)
if not flag_group:
qs = qs.narrow('%s:"%s"' %
('groups_exact', qs.query.clean('public')))
self.facets.append(
get_facet_by_name(param_facet_url,
qs.facet_counts()['fields'].items(),
'categories'))
return qs
def build_form(self, *args, **kwargs):
sqs = SearchQuerySet()
sqs = sqs.narrow('instance:"%s"' % self.request.instance.label)
self.searchqueryset = sqs
return super(InstanceSearchView, self).build_form(*args, **kwargs)
class SearchQuerySetTestCase(TestCase):
fixtures = ["base_data.json", "bulk_data.json"]
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections["default"]._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.uuidmmsi = SimpleMockUUIDModelIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi, self.uuidmmsi])
connections["default"]._index = self.ui
# Update the "index".
backend = connections["default"].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections["default"]._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object"
r" at 0x[0-9A-Fa-f]+>, using=None>$",
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in iter(msqs)]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections["default"].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(
[int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]],
)
self.assertEqual(len(connections["default"].queries), 1)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections["default"].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(
check,
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
],
)
self.assertEqual(len(connections["default"].queries), 3)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
# CharPK testing
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections["default"]._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, ["sometext", "1234"])
self.assertEqual(len(connections["default"].queries), 1)
# UUID testing
self.ui.build(indexes=[self.uuidmmsi])
connections["default"]._index = self.ui
self.uuidmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(
loaded,
[
"53554c58-7051-4350-bcc9-dad75eb248a9",
"77554c58-7051-4350-bcc9-dad75eb24888",
],
)
connections["default"]._index = old_ui
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = list(results)
self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections["default"].queries), 4)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue("foo" in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections["default"]._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections["default"]._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost("foo", 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling_override(self):
sqs = self.msqs.filter(content="not the spellchecking query")
self.assertEqual(sqs.query.spelling_query, None)
sqs = self.msqs.set_spelling_query("override")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.spelling_query, "override")
def test_spelling_suggestions(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content="Indx")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion("indexy"), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search("foo")), 23)
self.assertEqual(
len(
self.msqs.raw_search("(content__exact:hello AND content__exact:world)")
),
23,
)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend("charpk")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 2
)
# Models with uuid primary keys.
sqs = SearchQuerySet()
sqs.query.backend = UUIDMockSearchBackend("uuid")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 2
)
# If nothing is handled, you get nothing.
old_ui = connections["default"]._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections["default"]._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections["default"]._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections["default"]._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections["default"]._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 1
)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 2
)
# Restore.
connections["default"]._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query("test search -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND content__content=test search -stuff>",
)
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search -stuff>',
)
sqs = self.msqs.auto_query("test \"my thing\" search 'moar quotes' -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>",
)
sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>',
)
sqs = self.msqs.auto_query("test - stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), "<SQ: AND content__content=test - stuff>"
)
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), '<SQ: AND content__content="pants:rule">'
)
# Now with a different fieldname
sqs = self.msqs.auto_query("test search -stuff", fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), "<SQ: AND title__content=test search -stuff>"
)
sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND title__content=test "my thing" search -stuff>',
)
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet("foo").facet("bar")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="smarblaph",
)
self.fail()
except FacetingError as e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.",
)
sqs = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="month",
)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="month",
).date_facet(
"bar",
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="year",
)
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet("foo", "[bar TO *]")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet(
"bar", "[100 TO 499]"
)
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = (
self.msqs.query_facet("foo", "[bar TO *]")
.query_facet("bar", "[100 TO 499]")
.query_facet("foo", "[1000 TO 1499]")
)
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet("foo", "bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz")
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow("foo:moof")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo="bar", foo__lt="10")
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using="default")
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, "default")
def test_chaining(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content="bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter(content="oof")
sqs2 = self.msqs.filter(content="bar").filter(content="rab")
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, "OR")
self.assertEqual(
repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)
)
self.assertEqual(
repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter)
)
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter_or(content="oof")
sqs2 = self.msqs.filter(content="bar").filter_or(content="rab")
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, "AND")
self.assertEqual(
repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)
)
self.assertEqual(
repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter)
)
class SearchQuerySetTestCase(TestCase):
fixtures = ["base_data.json", "bulk_data.json"]
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections["default"]._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections["default"]._index = self.ui
# Update the "index".
backend = connections["default"].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections["default"]._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object" r" at 0x[0-9A-Fa-f]+>, using=None>$",
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in iter(msqs)]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections["default"].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]], [res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections["default"].queries), 1)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections["default"].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(
check,
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
],
)
self.assertEqual(len(connections["default"].queries), 3)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections["default"]._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, ["sometext", "1234"])
self.assertEqual(len(connections["default"].queries), 1)
connections["default"]._index = old_ui
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = list(results)
self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections["default"].queries), 4)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue("foo" in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections["default"]._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections["default"]._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost("foo", 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling_override(self):
sqs = self.msqs.filter(content="not the spellchecking query")
self.assertEqual(sqs.query.spelling_query, None)
sqs = self.msqs.set_spelling_query("override")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.spelling_query, "override")
def test_spelling_suggestions(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content="Indx")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion("indexy"), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search("foo")), 23)
self.assertEqual(len(self.msqs.raw_search("(content__exact:hello AND content__exact:world)")), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend("charpk")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# If nothing is handled, you get nothing.
old_ui = connections["default"]._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections["default"]._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections["default"]._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections["default"]._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections["default"]._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(len([result for result in results._result_cache if result is not None]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# Restore.
connections["default"]._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query("test search -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__content=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__content=test "my thing" search -stuff>')
sqs = self.msqs.auto_query("test \"my thing\" search 'moar quotes' -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), "<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>"
)
sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>',
)
sqs = self.msqs.auto_query("test - stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__content=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__content="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query("test search -stuff", fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND title__content=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND title__content=test "my thing" search -stuff>')
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet("foo").facet("bar")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet(
"foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="smarblaph"
)
self.fail()
except FacetingError as e:
self.assertEqual(
str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second."
)
sqs = self.msqs.date_facet(
"foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month"
)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet(
"foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month"
).date_facet("bar", start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="year")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet("foo", "[bar TO *]")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet("bar", "[100 TO 499]")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = (
self.msqs.query_facet("foo", "[bar TO *]")
.query_facet("bar", "[100 TO 499]")
.query_facet("foo", "[1000 TO 1499]")
)
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet("foo", "bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz")
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow("foo:moof")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo="bar", foo__lt="10")
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using="default")
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, "default")
def test_chaining(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content="bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter(content="oof")
sqs2 = self.msqs.filter(content="bar").filter(content="rab")
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, "OR")
self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter))
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter_or(content="oof")
sqs2 = self.msqs.filter(content="bar").filter_or(content="rab")
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, "AND")
self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter))
def _parse_facets(qd):
"""
Given a request, returns a quadruple of:
* A SearchQuerySet constrained by the request.GET vars.
* a dict of constraints that have been applied
* a list of potential choices for new constraints
* a querystring that produces these constraints (without pagination
or sorting)
"""
sqs = SearchQuerySet()
constraints = []
# Process constraints, and execute facets
for name, field in settings.ALL_FIELDS.iteritems():
if field['document']:
terms = qd.get(field['name'], qd.get('document_', None))
if terms:
sqs = sqs.auto_query(terms).highlight()
constraints.append({
'value': terms,
'key': name,
'type': 'document',
'display': field['display_name'],
})
elif field['faceted']:
sqs = sqs.raw_params(**{
'f.%s_exact.facet.limit' % field['name']: field['facet_limit']
})
if field['type'] == 'date':
sqs, start, end = _narrow_range(sqs, field, qd)
if start:
constraints.append({
'value': start.strftime("%Y-%m-%d"),
'key': name + '__gte',
'type': 'date',
'display': "%s - more than" % field['display_name'],
})
if end:
constraints.append({
'value': end.strftime("%Y-%m-%d"),
'key': name + '__lte',
'type': 'date',
'display': "%s - less than" % field['display_name'],
})
if start or end or name in settings.FACET_DISPLAY:
if not end:
end = utils.d_to_dt(getattr(SearchQuerySet().order_by("-" + name)[0], name))
if not start:
start = utils.d_to_dt(getattr(SearchQuerySet().order_by(name)[0], name))
span = max(1, (end - start).days)
gap = max(1, int(span / 100))
sqs = sqs.date_facet(name, start, end, 'day', gap)
elif field['type'] in ('int', 'float', 'latitude', 'longitude'):
sqs, start, end = _narrow_range(sqs, field, qd)
if start:
constraints.append({
'value': start,
'key': name + '__gte',
'type': 'min_max',
'display': "%s - more than" % field['display_name'],
})
if end:
constraints.append({
'value': end,
'key': name + '__lte',
'type': 'min_max',
'display': "%s - less than" % field['display_name'],
})
if start or end or name in settings.FACET_DISPLAY:
sqs = sqs.facet(name)
# if not end:
# end = getattr(SearchQuerySet().order_by("-" + name)[0], name)
# if not start:
# start = getattr(SearchQuerySet().order_by(name)[0], name)
# span = max(1, (end - start))
# gap = max(1, int(span / 100))
# exact = "%s_exact" % name
# sqs = sqs.raw_params(**{
# "facet.range": exact,
# "f.%s.facet.range.start" % exact: start,
# "f.%s.facet.range.end" % exact: end,
# "f.%s.facet.range.gap" % exact: gap,
# })
else:
val = qd.get(name, None)
if val is not None:
constraints.append({
'value': val,
'key': name,
'type': 'text',
'display': field['display_name'],
})
sqs = sqs.narrow('%s:"%s"' % (name + "_exact", val))
if val is not None or name in settings.FACET_DISPLAY:
sqs = sqs.facet(name)
# Constraint removal links
queryargs = dict((p['key'], p['value']) for p in constraints)
querystr = urllib.urlencode(queryargs)
for param in constraints:
# remove key
value = queryargs.pop(param['key'])
remainder = urllib.urlencode(queryargs)
param.update({'remove_link': remainder})
# restore key
queryargs[param['key']] = value
# Choices
counts = sqs.facet_counts()
choices = []
for name in settings.FACET_DISPLAY:
field = settings.ALL_FIELDS[name]
choice = {'field': field}
if field['document']:
choice.update({
'type': 'document',
'value': queryargs.get(name, ''),
})
elif field['type'] in ('text', 'char'):
facets = sorted((k, c) for k, c in counts['fields'][name] if c > 0)
if facets:
choice.update({
'choices': facets,
'type': 'text',
'value': queryargs.get(name, '')
})
elif field['type'] in ('int', 'float', 'latitude', 'longitude'):
facets = sorted([(int(k), c) for k,c in counts['fields'][name] if c > 0])
if facets:
choice.update({
'type': 'min_max',
'counts': [c for k,c in facets],
'vals': [k for k,c in facets],
'min_value': facets[0][0],
'max_value': facets[-1][0],
'chosen_min': queryargs.get(name + '__gte', ''),
'chosen_max': queryargs.get(name + '__lte', ''),
})
elif field['type'] == 'date':
facets = sorted(counts['dates'].get(name, {}).iteritems())
if facets:
val_counts = []
vals = []
last_dt = None
for d,c in facets:
if c > 0:
try:
last_dt = utils.iso_to_datetime(d)
val_counts.append(c)
vals.append(last_dt.strftime('%Y-%m-%d'))
except (TypeError, ValueError):
pass
if vals and last_dt:
max_value = min(
utils.iso_to_datetime(counts['dates'][name]['end']),
utils.d_to_dt(
getattr(SearchQuerySet().order_by('-' + name)[0], name)
) + datetime.timedelta(days=1),
last_dt + datetime.timedelta(
days=int(re.sub('[^\d]', '', counts['dates'][name]['gap']))
),
)
vals.append(max_value.strftime('%Y-%m-%d'))
val_counts.append(0)
choice.update({
'type': 'date',
'counts': val_counts,
'vals': vals,
'min_value': vals[0],
'max_value': vals[-1],
'chosen_min': queryargs.get(name + '__gte', ''),
'chosen_max': queryargs.get(name + '__lte', ''),
})
choices.append(choice)
return sqs, choices, constraints, querystr
