def search(self): sqs = SearchQuerySet().load_all() if not self.is_valid(): return sqs if self.cleaned_data['q']: # search input of a code, contains at least '_YYYY_' # if not re.match('([a-zA-Z]*_?[EI])?_[0-9]{4}_([0-9]{3}_[0-9]{3})?', self.cleaned_data.get('q')): sqs = sqs.filter(content__startswith=self.cleaned_data['q']).facet( 'item_acces').facet('item_status').facet('digitized').facet( 'recording_context').facet('physical_format').facet( 'media_type') # else: # sqs = sqs.filter(code__contains=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type') for facet in self.selected_facets: if ":" not in facet: continue field, value = facet.split(":", 1) if value: if value == 'viewable': sqs = sqs.narrow('item_acces:full OR item_acces:mixed' ).narrow('digitized:T') else: sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value))) return sqs
def search(self): sqs = SearchQuerySet().load_all() if not self.is_valid(): return sqs if self.cleaned_data['q']: #search input of a code, contains at least '_YYYY_' #if not re.match('([a-zA-Z]*_?[EI])?_[0-9]{4}_([0-9]{3}_[0-9]{3})?', self.cleaned_data.get('q')): sqs = sqs.filter(content__startswith=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type') #else: # sqs = sqs.filter(code__contains=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type') for facet in self.selected_facets: if ":" not in facet: continue field, value = facet.split(":", 1) if value: if value == 'viewable': sqs = sqs.narrow('item_acces:full OR item_acces:mixed').narrow('digitized:T') else: sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value))) return sqs
def myitems_index(request, view_name, page_title, no_items_message, index_name, only_published=True, template="myitems/index.html"): breadcrumbs = [ {"url": reverse("myitems:myitems"), "title": u"My Items"}, {"url": reverse("myitems:%s" % view_name), "title": page_title}, ] query_string_params = {} index_params = IndexParams(request) query_string_params = index_params.update_query_string_params(query_string_params) batch_end = index_params.batch_start + index_params.batch_size query = SearchQuerySet() if only_published: query = query.narrow("is_displayed:true") query = query.narrow("%s:%i" % (index_name, request.user.id)) if index_params.query_order_by is not None: query = query.order_by(index_params.query_order_by) items = [] results = query[index_params.batch_start:batch_end] for result in results: items.append(populate_item_from_search_result(result)) pagination = Pagination(request.path, query_string_params, index_params.batch_start, index_params.batch_size, len(query)) return direct_to_template(request, template, locals())
def search_api(request): query = request.REQUEST.get("q", "") start = int(request.REQUEST.get("start", 0)) limit = int(request.REQUEST.get("limit", getattr(settings, "HAYSTACK_SEARCH_RESULTS_PER_PAGE", 25))) sort = request.REQUEST.get("sort", "relevance") type = request.REQUEST.get("bytype") sqs = SearchQuerySet() if type is not None: if type in ["map", "layer", "contact", "group"]: # Type is one of our Major Types (not a sub type) sqs = sqs.narrow("type:%s" % type) elif type in ["vector", "raster"]: # Type is one of our sub types sqs = sqs.narrow("subtype:%s" % type) if query: sqs = sqs.filter(content=AutoQuery(query)) sqs = sqs.facet("type").facet("subtype") if sort.lower() == "newest": sqs = sqs.order_by("-date") elif sort.lower() == "oldest": sqs = sqs.order_by("date") elif sort.lower() == "alphaaz": sqs = sqs.order_by("title") elif sort.lower() == "alphaza": sqs = sqs.order_by("-title") results = [] for i, result in enumerate(sqs[start:start + limit]): data = json.loads(result.json) data.update({"iid": i + start}) results.append(data) facets = sqs.facet_counts() counts = {"map": 0, "layer": 0, "vector": 0, "raster": 0, "contact": 0, "group": 0} for t, c in facets.get("fields", {}).get("type", []): counts[t] = c for t, c in facets.get("fields", {}).get("subtype", []): counts[t] = c data = { "success": True, "total": sqs.count(), "rows": results, "counts": counts, } return HttpResponse(json.dumps(data), mimetype="application/json")
def search(self): sqs = SearchQuerySet() sqs = sqs.facet('bureaux').facet('section').facet('annee').facet( 'partenaire') if self.q: sqs = sqs.filter(content=sqs.query.clean(self.q)) # if self.courant: # sqs = sqs.filter(date_fin__gte=datetime.date.today()) # if self.cloture: # sqs = sqs.filter(date_fin__lt=datetime.date.today()) self.selected_facets = list( set(self.selected_facets.split('&') + self.selected_facets_get)) for facet in self.selected_facets: if "__" not in facet: continue field, value = facet.split("__", 1) if value: sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value))) return sqs
def get_searchqueryset(self, form): """Get the Haystack searchqueryset (which we treat as a regular Django queryset.""" sqs = SearchQuerySet() if self.model: sqs = sqs.models(self.model) # FIXME: Move somewhere more sensible if settings.PORTAL_HIDE_DRAFTS and not self.request.user.is_staff: sqs = sqs.narrow("publication_status:%d" % models.Resource.PUBLISHED) for facet in self.facetclasses: sqs = facet.apply(sqs) # apply the query if form.is_valid(): sqs = form.filter(sqs) for facetclass in self.facetclasses: sqs = facetclass.narrow(sqs, self.request.GET.getlist( facetclass.paramname)) counts = sqs.facet_counts() current = sqs.query.narrow_queries for facetclass in self.facetclasses: facetclass.parse(counts, current) # FIXME: Find way around assigning the sqs to the instance, # but it seems to be difficult to prevent it from running # multiple times otherwise, e.g. when checking for a spelling # suggestion. self.searchqueryset = sqs return sqs
def get_items(self, from_date=None, until_date=None, set=None, microsite=None): query = SearchQuerySet().narrow("is_displayed:true") if from_date: query = query.filter(published_on__gte=from_date) if until_date: query = query.filter(published_on__lte=until_date) if set is not None: set_name, slug = set.split(":") if set_name == "collection": query = query.narrow("collection:%i" % Collection.objects.get(slug=slug).id) if microsite: query = query.narrow("microsites:%i" % microsite.id) return query.order_by("published_on").load_all()
def get_searchqueryset(self, form): """Get the Haystack searchqueryset (which we treat as a regular Django queryset.""" sqs = SearchQuerySet() if self.model: sqs = sqs.models(self.model) # FIXME: Move somewhere more sensible if settings.PORTAL_HIDE_DRAFTS and not self.request.user.is_staff: sqs = sqs.narrow("publication_status:%d" % models.Resource.PUBLISHED) for facet in self.facetclasses: sqs = facet.apply(sqs) # apply the query if form.is_valid(): sqs = form.filter(sqs) for facetclass in self.facetclasses: sqs = facetclass.narrow( sqs, self.request.GET.getlist(facetclass.paramname)) counts = sqs.facet_counts() current = sqs.query.narrow_queries for facetclass in self.facetclasses: facetclass.parse(counts, current) # FIXME: Find way around assigning the sqs to the instance, # but it seems to be difficult to prevent it from running # multiple times otherwise, e.g. when checking for a spelling # suggestion. self.searchqueryset = sqs return sqs
def get_context(self, request): context = super(RecordIndexPage, self).get_context(request) # Get selected facets selected_facets = set(request.GET.getlist('selected_facets')) # Init a search query set sqs = SearchQuerySet().models(RecordPage) # Apply currently selected facets for facet in selected_facets: sqs = sqs.narrow(facet) # Get facet counts sqs = sqs.facet('language').facet('word_type').facet('first_letter') # Generate presentable facet data selected_facets_ui = [] for facet in selected_facets: f = { 'value': facet.split(':')[1], 'remove_url': request.get_full_path().replace( '&selected_facets={}'.format(facet), '') } selected_facets_ui.append(f) context['selected_facets'] = selected_facets_ui context['sqs'] = sqs return context
def get_queryset(self, query_args): # Create new queryset qs = SearchQuerySet() # Are we searching all models or just a specific one (depends on # parameter set in View instantiation) if self.search_model is not None: qs = qs.models(self.search_model) # Do we have a query or are we just getting all of them? if 'q' in query_args: qry = query_args['q'] ## Currently deactivated due to policy decision to allow the users ## themselves to choose the granularity and fuzzyness of the search ## # fuzzify search # qry = u'{}~'.format(qry.replace(' ', '~ ')) qs = qs.auto_query(qry) elif 'parl_id' in query_args: qs = qs.filter(parl_id=query_args['parl_id']) if 'llp_numeric' in query_args: qs = qs.filter(llps_numeric=query_args['llp_numeric']) return (qs.all(), None) # Filter by facets if query_args['facet_filters']: for facet_field in query_args['facet_filters'].keys(): # We use narrow to limit the index entries beforehand, but # need to use filter afterwards to remove partly correct results # For instance, searching for Steyr (Oberoesterreich) yielded # everyone from Oberoesterreich until filtering by it again. qs = qs.narrow(u"{}:{}".format( facet_field, qs.query.clean(query_args['facet_filters'][facet_field]) )).filter( **{facet_field: query_args['facet_filters'][facet_field]}) # Retrieve facets and facet_counts facet_counts = [] if self.facet_fields: facets = qs for facet_field in self.facet_fields: if self.facet_fields[facet_field]['type'] == 'date': facets = facets.date_facet( facet_field, start_date=datetime.date(1900, 1, 1), end_date=datetime.date(2050, 1, 1), gap_by='month') if self.facet_fields[facet_field]['type'] == 'field': facets = facets.facet(facet_field) facet_counts = facets.facet_counts() # Get results and return them if 'only_facets' in query_args: result = {} else: result = qs.all() return (result, facet_counts)
def get_queryset(cls, user): queryset = SearchQuerySet() queryset = queryset.models(*CREATED_MODELS) queryset = queryset.narrow("is_displayed:true") queryset = queryset.filter(SQ(creator=user.id)) return queryset
def search(self): sqs = SearchQuerySet() sqs = sqs.facet('bureaux').facet( 'section').facet('annee').facet('partenaire') if self.q: sqs = sqs.filter(content=sqs.query.clean(self.q)) if self.courant: sqs = sqs.filter(date_fin__gte=datetime.date.today()) if self.cloture: sqs = sqs.filter(date_fin__lt=datetime.date.today()) self.selected_facets = list( set(self.selected_facets.split('&') + self.selected_facets_get)) for facet in self.selected_facets: if "__" not in facet: continue field, value = facet.split("__", 1) if value: sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value))) return sqs
def get_object(self, request): self.queryDict = request.GET all_results = SearchQuerySet().all() facets = None if 'selected_facets' in request.GET: facets = request.GET.getlist('selected_facets') if 'q' in request.GET: self.query = request.GET['q'] results = all_results.filter(content=self.query) if facets: for facet in facets: (facet_name, facet_value) = facet.split(':') facet_name = facet_name.rsplit('_exact')[0] results = results.narrow('%s:%s' % (facet_name, facet_value)) elif facets: for facet in facets: (facet_name, facet_value) = facet.split(':') facet_name = facet_name.rsplit('_exact')[0] results = all_results.narrow('%s:%s' % (facet_name, facet_value)) return results.order_by('-last_action_date')
def get_near_items(item): #import ipdb; ipdb.set_trace() qs = SearchQuerySet().all() loc = item.locations.first() point = Point(loc.lng, loc.lat) qs = qs.dwithin('location', point, D(m=50000)).distance('location', point) qs = qs.narrow('%s:"%s"' % ('groups_exact', 'public')) return qs
def get_facets_for_field(field, model=None): query = SearchQuerySet() if model is not None: query = query.models(model) query = query.narrow("is_displayed:true") query = query.facet(field) return query.facet_counts().get("fields", {}).get(field, [])
def microsite(request, microsite): microsite = get_object_or_404(Microsite, slug=microsite) page_title = u"%s Home" % microsite.name breadcrumbs = [{"url": reverse("materials:microsite", kwargs=dict(microsite=microsite.slug)), "title": page_title}] query = SearchQuerySet().narrow("is_displayed:true") query = query.narrow("microsites:%i" % microsite.id) query = query.order_by("-rating") query = query.facet("indexed_topics").facet("keywords").facet("grade_levels").facet("course_material_types") items = [] results = query[0:8] for result in results: items.append(populate_item_from_search_result(result)) facets = query.facet_counts()["fields"] topics = [] topic_counts = dict(facets["indexed_topics"]) for topic, tree_info in tree_item_iterator(microsite.topics.all()): topic.count = topic_counts.get(str(topic.id), 0) topics.append((topic, tree_info)) grade_levels = [] grade_level_counts = dict(facets["grade_levels"]) for level in GradeLevel.objects.all(): level.count = grade_level_counts.get(str(level.id), 0) grade_levels.append(level) course_material_types = [] course_material_type_counts = dict(facets["course_material_types"]) for material_type in CourseMaterialType.objects.all(): material_type.count = course_material_type_counts.get(str(material_type.id), 0) course_material_types.append(material_type) keywords = query.count() and facets.get("keywords", []) or [] if len(keywords) > MAX_TOP_KEYWORDS: keywords = keywords[:MAX_TOP_KEYWORDS] keywords = get_tag_cloud(dict(keywords), 3, 0, 0) for keyword in keywords: name = get_name_from_slug(Keyword, keyword["slug"]) or \ get_name_from_slug(Tag, keyword["slug"]) or \ keyword["slug"] keyword["name"] = name featured_k12 = SearchQuerySet().filter(workflow_state=PUBLISHED_STATE, featured=True, grade_levels__in=(1, 2), microsites=microsite.id).order_by("-featured_on").load_all()[:3] featured_k12 = [r.object for r in featured_k12 if r] featured_highered = SearchQuerySet().filter(workflow_state=PUBLISHED_STATE, featured=True, grade_levels=3, microsites=microsite.id).order_by("-featured_on").load_all()[:3] featured_highered = [r.object for r in featured_highered if r] slides = Slide.objects.filter(microsite=microsite) resource_number = SearchQuerySet().filter(workflow_state=PUBLISHED_STATE, microsites=microsite.id).count() return direct_to_template(request, "materials/microsites/%s.html" % microsite.slug, locals())
def get_queryset(self, query_args): # Create new queryset qs = SearchQuerySet() # Are we searching all models or just a specific one (depends on # parameter set in View instantiation) if self.search_model is not None: qs = qs.models(self.search_model) # Do we have a query or are we just getting all of them? if 'q' in query_args: qry = query_args['q'] # fuzzify search qry = u'{}~'.format(qry.replace(' ', '~ ')) qs = qs.auto_query(qry) elif 'parl_id' in query_args: qs = qs.filter(parl_id=query_args['parl_id']) if 'llp_numeric' in query_args: qs = qs.filter(llps_numeric=query_args['llp_numeric']) return (qs.all(), None) # Filter by facets if query_args['facet_filters']: for facet_field in query_args['facet_filters'].keys(): # We use narrow to limit the index entries beforehand, but # need to use filter afterwards to remove partly correct results # For instance, searching for Steyr (Oberoesterreich) yielded # everyone from Oberoesterreich until filtering by it again. qs = qs.narrow(u"{}:{}".format( facet_field, qs.query.clean(query_args['facet_filters'][facet_field])) ).filter( **{ facet_field: query_args['facet_filters'][facet_field]} ) # Retrieve facets and facet_counts facet_counts = [] if self.facet_fields: facets = qs for facet_field in self.facet_fields: if self.facet_fields[facet_field]['type'] == 'date': facets = facets.date_facet( facet_field, start_date=datetime.date(1900, 1, 1), end_date=datetime.date(2050, 1, 1), gap_by='month') if self.facet_fields[facet_field]['type'] == 'field': facets = facets.facet(facet_field) facet_counts = facets.facet_counts() # Get results and return them if 'only_facets' in query_args: result = {} else: result = qs.all() return (result, facet_counts)
def projectsheet_search(self, request, **kwargs): """ Search project sheets. """ self.method_check(request, allowed=['get']) self.throttle_check(request) self.is_authenticated(request) # Query params query = request.GET.get('q', '') autocomplete = request.GET.get('auto', None) selected_facets = request.GET.getlist('facet', None) order = request.GET.getlist('order', None) sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags') # narrow down QS with facets if selected_facets: for facet in selected_facets: sqs = sqs.narrow('tags:%s' % (facet)) # A: if autocomplete, we return only a list of tags # starting with "auto" along with their count. if autocomplete is not None: tags = sqs.facet_counts() tags = tags['fields']['tags'] if len(autocomplete) > 0: tags = [t for t in tags if t[0].startswith(autocomplete)] tags = [{'name': t[0], 'count': t[1]} for t in tags] object_list = { 'objects': tags, } # B: else, we return a list of projectsheets else: # launch query if query != "": sqs = sqs.auto_query(query) uri = reverse('api_projectsheet_search', kwargs={'api_name': self.api_name, 'resource_name': self._meta.resource_name}) paginator = Paginator(request.GET, sqs, resource_uri=uri) objects = [] for result in paginator.page()['objects']: if result: bundle = self.build_bundle(obj=result.object, request=request) bundle = self.full_dehydrate(bundle, for_list=True) objects.append(bundle) object_list = { 'meta': paginator.page()['meta'], 'objects': objects, } self.log_throttled_access(request) return self.create_response(request, object_list)
def search(self): if not self.is_valid(): return self.no_query_found() sqs = SearchQuerySet().models(Port).facet('maintainers').facet( 'categories').facet('variants') # Performing narrowing based on facets for facet in self.selected_facets: if ":" not in facet: continue field, value = facet.split(":", 1) if value: sqs = sqs.narrow('%s:"%s"' % (field, sqs.query.clean(value))) sort_by = "name_lower" # Filter out deleted ports, based on query if not self.cleaned_data.get('show_deleted_ports'): sqs = sqs.filter(active=True) # If a search query is present, only then perform the search operations if self.cleaned_data.get('q'): if self.cleaned_data['name']: sqs = sqs.filter(name=self.cleaned_data['q']) sort_by = "name_length" else: sqs = sqs.filter( SQ(name=self.cleaned_data['q']) | SQ(description=self.cleaned_data['q'])) sqs = sqs.order_by(sort_by) # Filter operations, perform even if a search query is absent # This is done to allow viewing all "outdated ports", "all ports with broken livecheck" etc. f = SQ() if self.cleaned_data['livecheck_uptodate']: f = SQ(livecheck_broken=False) & SQ(livecheck_outdated=False) if self.cleaned_data['livecheck_broken']: f = f | SQ(livecheck_broken=True) if self.cleaned_data['livecheck_outdated']: f = f | SQ(livecheck_outdated=True) if self.cleaned_data['nomaintainer']: f = f & SQ(nomaintainer=True) if self.cleaned_data['installed_file']: f = f & SQ(files=Exact(self.cleaned_data['installed_file'])) if f != SQ(): sqs = sqs.filter(f) return sqs
def green_browse(request): microsite = get_object_or_404(Microsite, slug="green") query = SearchQuerySet().narrow("is_displayed:true") query = query.narrow("microsites:%i" % microsite.id) query = query.facet("indexed_topics").facet("keywords").facet("grade_levels").facet("course_material_types") facets = query.facet_counts()["fields"] topics = [] topic_counts = dict(facets["indexed_topics"]) for topic, tree_info in tree_item_iterator(microsite.topics.all()): topic.count = topic_counts.get(str(topic.id), 0) topics.append((topic, tree_info)) grade_levels = [] grade_level_counts = dict(facets["grade_levels"]) for level in GradeLevel.objects.all(): level.count = grade_level_counts.get(str(level.id), 0) grade_levels.append(level) course_material_types = [] course_material_type_counts = dict(facets["course_material_types"]) for material_type in CourseMaterialType.objects.all(): material_type.count = course_material_type_counts.get(str(material_type.id), 0) course_material_types.append(material_type) keywords = query.count() and facets.get("keywords", []) or [] if len(keywords) > MAX_TOP_KEYWORDS: keywords = keywords[:MAX_TOP_KEYWORDS] keywords = get_tag_cloud(dict(keywords), 3, 0, 0) for keyword in keywords: name = get_name_from_slug(Keyword, keyword["slug"]) or \ get_name_from_slug(Tag, keyword["slug"]) or \ keyword["slug"] keyword["name"] = name query = SearchQuerySet().narrow("is_displayed:true") query = query.narrow("microsites:%i" % microsite.id) query = query.order_by("-published_on").load_all() recently_added = [r.object for r in query[:7]] return direct_to_template(request, "materials/microsites/green-browse.html", locals())
def file_search(self, request, **kwargs): """ Implement file searching. """ self.method_check(request, allowed=['get']) self.throttle_check(request) self.is_authenticated(request) # URL params bucket_id = kwargs['bucket_id'] # Query params query = request.GET.get('q', '') autocomplete = request.GET.get('auto', None) selected_facets = request.GET.getlist('facet', None) order = request.GET.getlist('order', '-pub_date') sqs = SearchQuerySet().models(BucketFile).filter( bucket=bucket_id).order_by(order).facet('tags') # 1st narrow down QS if selected_facets: for facet in selected_facets: sqs = sqs.narrow('tags:%s' % (facet)) # A: if autocomplete, we return only a list of tags starting # with "auto" along with their count. if autocomplete is not None: tags = sqs.facet_counts() tags = tags['fields']['tags'] if len(autocomplete) > 0: tags = [t for t in tags if t[0].startswith(autocomplete)] tags = [ {'name': t[0], 'count':t[1]} for t in tags ] object_list = { 'objects': tags, } # B: else, we return a list of files else: if query != "": sqs = sqs.auto_query(query) objects = [] # Building object list for result in sqs: bundle = self.build_bundle(obj=result.object, request=request) bundle = self.full_dehydrate(bundle) objects.append(bundle) object_list = { 'objects': objects, } self.log_throttled_access(request) return self.create_response(request, object_list)
def reindex_microsite_topic(topic): from haystack.query import SearchQuerySet objects = set() # get all objects from this topic and all objects with this topic's keywords query = "indexed_topics:%s" % topic.id for result in SearchQuerySet().narrow(query).load_all(): objects.add(result.object) topic_keywords = topic.keywords.values_list("slug", flat=True) microsite_keywords = topic.microsite.keywords.values_list("slug", flat=True) if topic_keywords and microsite_keywords: query = SearchQuerySet() query = query.narrow("keywords(%s)" % " OR ".join(["%s" % kw for kw in topic_keywords])) query = query.narrow("keywords(%s)" % " OR ".join(["%s" % kw for kw in microsite_keywords])) for result in query.load_all(): objects.add(result.object) for instance in objects: reindex(instance)
def ms_search(self, request, **kwargs): self.method_check(request, allowed=['get']) self.throttle_check(request) self.is_authenticated(request) get_params = request.GET.copy() if "advanced" in get_params: return self.ms_advanced_search(request, **kwargs) # Query params query = get_params.get('q', None) selected_facets = [ item for sublist in [f.split(',') for f in get_params.getlist('facet', "")] for item in sublist ] selected_facets = None if selected_facets == [''] else selected_facets ordering = get_params.get('ordering', None) limit = get_params.get('limit', self._meta.limit) for word in ["q", "facet", "ordering", "format", 'limit', 'offset']: if word in get_params.keys(): try: del get_params[word] except KeyError: print word, "not in request" filtering = {} for key, val in get_params.iteritems(): filtering[key] = json.loads(val) sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags') if selected_facets: first_narrow_succes = False for i, facet in enumerate(selected_facets): tmp_sqs = sqs.narrow('tags_exact:%s' % slugify(facet)) if len(tmp_sqs): first_narrow_succes = True if first_narrow_succes or i == len(selected_facets) - 1: sqs = tmp_sqs if query: sqs = sqs.filter(text=Raw(query)) if filtering: sqs = sqs.filter(**filtering) if ordering: sqs = sqs.order_by(ordering) self.log_throttled_access(request) return self.create_response(request, self.prepare_result(request, sqs, limit))
def search(self): kwargs = { "hl.simple.pre": "<mark>", "hl.simple.post": "</mark>", "hl.fragsize": 400, "hl.snippets": 3, } sqs = self.no_query_found() # use the text_exact field in solr which doesn't stem if self.cleaned_data["q"] and self.cleaned_data["exact"] is True: query = self.cleaned_data["q"] kwargs["hl.fl"] = "text_exact" sqs = SearchQuerySet().filter(text_exact=Exact(query)) if self.load_all: sqs = sqs.load_all() else: sqs = super(HighlightedSearchForm, self).search() if self.cleaned_data["similar_type"]: similar_council = self.cleaned_data["similar_council"] comparison_type = self.cleaned_data["similar_type"] cut_off = 15 names = Distance.objects.filter( council_a=similar_council, type=comparison_type, position__lte=cut_off, ).values_list("council_b__name", flat=True) query_names = [f'council_name:"{name}"' for name in names] sqs = sqs.narrow(" OR ".join(query_names)) if self.cleaned_data["council_name"]: # narrow makes use of fq rather than q sqs = sqs.narrow("council_name:%s" % self.cleaned_data["council_name"]) return sqs.highlight(**kwargs)
def results(request): url = urllib.unquote_plus(request.get_full_path()) print url searchform = SearchForm(request.GET) if not "q" in request.GET: return redirect('search.views.index') if searchform.is_valid(): q = searchform.cleaned_data['q'] selected_facets = request.GET.getlist("selected_facets") remove_facets = request.GET.getlist("remove_facets") applied_facets = {} if remove_facets: for r in remove_facets: filter(lambda a: a != r, selected_facets) filter(lambda a: a != r, applied_facets) if selected_facets: for f in selected_facets: k, v = f.split(":") myurl = url.replace("&selected_facets=%s" % f, "") tag = "%s : %s" % (k.upper(), v) applied_facets[tag] = myurl print applied_facets if q: sqs = SearchQuerySet().models(Part).models(Company).facet('brand').facet('category').facet('with_distributors').facet('with_image').auto_query(q) for facet in selected_facets: sqs = sqs.narrow(facet) try: page = request.GET.get('page', 1) except PageNotAnInteger: page = 1 p = Paginator(sqs, 10, request=request) results_list = p.page(page) return render_to_response('search/results.html', { 'results_list': results_list, 'query': q, 'facets': sqs.facet_counts(), 'applied_facets': applied_facets, }, context_instance=RequestContext(request))
def ms_search(self, request, **kwargs): self.method_check(request, allowed=['get']) self.throttle_check(request) self.is_authenticated(request) get_params = request.GET.copy() if "advanced" in get_params: return self.ms_advanced_search(request, **kwargs) # Query params query = get_params.get('q', None) selected_facets = [item for sublist in [f.split(',') for f in get_params.getlist('facet', "")] for item in sublist] selected_facets = None if selected_facets == [''] else selected_facets ordering = get_params.get('ordering', None) limit = get_params.get('limit', self._meta.limit) for word in ["q", "facet", "ordering", "format", 'limit', 'offset']: if word in get_params.keys(): try: del get_params[word] except KeyError: print word, "not in request" filtering = {} for key, val in get_params.iteritems(): filtering[key] = json.loads(val) sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags') if selected_facets: first_narrow_succes = False for i, facet in enumerate(selected_facets): tmp_sqs = sqs.narrow('tags_exact:%s' % slugify(facet)) if len(tmp_sqs) : first_narrow_succes = True if first_narrow_succes or i == len(selected_facets) - 1: sqs = tmp_sqs if query: sqs = sqs.filter(text=Raw(query)) if filtering : sqs = sqs.filter(**filtering) if ordering: sqs = sqs.order_by(ordering) self.log_throttled_access(request) return self.create_response(request, self.prepare_result(request, sqs, limit))
def get_facet_counts(self): queryset = SearchQuerySet().models(PostAssertion) queryset = self._apply_facets_to_queryset(queryset) params = self.request.GET if ('date_from' in params and params['date_from']) or \ ('date_to' in params and params['date_to']): query = 'date:[{} TO {}]'.format( -1 * int(params['date_from']) if 'date_from' in params and params['date_from'] else PromrepFacetedSearchForm.MIN_DATE, -1 * int(params['date_to']) if 'date_to' in params and params['date_to'] else PromrepFacetedSearchForm.MAX_DATE ) queryset = queryset.narrow(query) for field in PromrepFacetedSearchForm.AUTOCOMPLETE_FACETS: if field in params and params[field]: queryset = queryset.narrow( '{}:{}'.format(field, params[field])) return queryset.facet_counts()
def get_facets_for_field(field, model=None, facet_limit=None, facet_mincount=None): query = SearchQuerySet() if model is not None: if isinstance(model, (tuple, list)): query = query.models(*model) else: query = query.models(model) query = query.narrow("is_displayed:true") query = query.facet(field) if facet_limit: query = query.facet_limit(facet_limit) if facet_mincount: query = query.facet_mincount(facet_mincount) return query.facet_counts().get("fields", {}).get(field, [])
def featured_counts(obj): try: featured_counts = {} sqs = SearchQuerySet() sqs = sqs.facet("featured") sqs = sqs.narrow("featured:[* TO *]") for featured in sqs.facet_counts()['fields']['featured']: key = featured[0].replace(' ', '') key = key.replace('-', '') featured_counts[key] = featured[1] return featured_counts except: return None
def featured_counts(obj): try: featured_counts = {} sqs = SearchQuerySet() sqs = sqs.facet("featured") sqs = sqs.narrow("featured:[* TO *]") for featured in sqs.facet_counts()['fields']['featured']: key = featured[0].replace(' ', '') key = key.replace('-', '') featured_counts[key] = featured[1] return featured_counts except KeyError: return None
def get_queryset(self): queryset = super(SenateSearchView, self).get_queryset() if 'senate_date' in self.request.GET: queryset = SearchQuerySet().models(StatusAssertion) else: queryset = queryset.narrow('date:[{0} TO {0}]'.format( SenateSearchForm.INITIAL_DATE)) certainty = self.request.GET.get('dating_certainty') if certainty is not None and certainty == '3': return queryset.order_by('-date_end') return queryset.order_by('date_start')
def facet_counts(obj, facetName): try: counts = {} sqs = SearchQuerySet() sqs = sqs.facet(facetName) sqs = sqs.narrow(facetName + ":[* TO *]") for x in sqs.facet_counts()['fields'][facetName]: key = x[0].replace(' ', '') key = key.replace('-', '') counts[key] = x[1] return counts except: return None
def infrastructure_counts(obj): try: infrastructure_counts = {} sqs = SearchQuerySet() sqs = sqs.facet("infrastructure") sqs = sqs.narrow("infrastructure:[* TO *]") for infrastructure in sqs.facet_counts()['fields']['infrastructure']: key = infrastructure[0].replace(' ', '') key = key.replace('-', '') infrastructure_counts[key] = infrastructure[1] return infrastructure_counts except: return None
def ask_me_anything(request): context = {} if request.method == 'POST': q = request.POST.get('q', '') if q: q = Question(q) sq = SearchQuerySet() if q.get_type() in ('LOC', 'DATETIME', 'NUM', 'PERSON'): sq = sq.narrow('ann:' + q.get_type()) for k in q.weight_keywords(): sq = sq.filter_or(SQ(content=k) | SQ(title=k) | SQ(context=k)) context = { 'q': q, 'result': sq[:10], } return render(request, 'ask_me_anything.html', context)
def infrastructure_counts(obj): try: infrastructure_counts = {} sqs = SearchQuerySet() sqs = sqs.facet("infrastructure") sqs = sqs.narrow("infrastructure:[* TO *]") for infrastructure in sqs.facet_counts()['fields']['infrastructure']: key = infrastructure[0].replace(' ', '') key = key.replace('-', '') infrastructure_counts[key] = infrastructure[1] return infrastructure_counts except KeyError: return None
def facet_counts(obj, facetName): try: counts = {} sqs = SearchQuerySet() sqs = sqs.facet(facetName) sqs = sqs.narrow(facetName + ":[* TO *]") for x in sqs.facet_counts()['fields'][facetName]: key = x[0].replace(' ', '') key = key.replace('-', '') counts[key] = x[1] return counts except KeyError: return None
def bulk_payment_ajax_search(request): sqs = SearchQuerySet() sqs = sqs.narrow('content_type_exact:"Bulk Payment"') if 'q' in request.GET: q = request.GET['q'] sqs = sqs.filter(**{'mixed_name__startswith': q}) return [{ 'text': "(#%d) %s %d%s %s" % ( bulk_payment.object.pk, bulk_payment.object.person.mixed_name, bulk_payment.amount, bulk_payment.currency, bulk_payment.receipt_date.strftime("(%B %-d %Y)"), ), 'value': bulk_payment.object.pk } for bulk_payment in sqs]
def list(self): from haystack.query import SearchQuerySet results = SearchQuerySet() for facet in Facet.asset_facets: results = results.facet(facet) types = self.search_criteria.split(';') for t in types: criteria = t.split(',') q = '' for c in criteria: if len(q): q += ' OR ' q += c.strip() results = results.narrow(q) return results.order_by("name")
def get_queryset(self): queryset = SearchQuerySet().filter(active=True).order_by('first_name') # further filter queryset based on some set of criteria party = self.request.GET.getlist('party', '') party_or = "" college = self.request.GET.getlist('college', '') college_or = "" state = self.request.GET.getlist('state', '') state_or = "" issues = self.request.GET.getlist('issues', '') issues_or = "" race = self.request.GET.getlist('race', '') race_or = "" race_type = self.request.GET.getlist('race_type', '') race_type_or = "" women = self.request.GET.get('women', '') q = self.request.GET.get('q', '') page = self.request.GET.get('page', '') search = ''.join(party) + q + page if search == "": search = "null" if party: for facet in party: party_or += 'party: "%s"' % (facet) queryset = queryset.narrow(party_or) if college: for facet in college: college_or += 'college: "%s"' % (facet) queryset = queryset.narrow(college_or) if state: for facet in state: state_or += 'state: "%s"' % (facet) queryset = queryset.narrow(state_or) if issues: for facet in issues: issues_or += 'issues: "%s"' % (facet) queryset = queryset.narrow(issues_or) if race: for facet in race: race_or += 'race: "%s"' % (facet) queryset = queryset.narrow(race_or) if race_type: for facet in race_type: race_type_or += 'race_type: "%s"' % (facet) queryset = queryset.narrow(race_type_or) if q: queryset = queryset.filter( SQ(text=AutoQuery(q)) | SQ(title=AutoQuery(q))) if women: queryset = queryset.filter(women=False) return queryset
def projectsheet_search(self, request, **kwargs): self.method_check(request, allowed=['get']) self.throttle_check(request) self.is_authenticated(request) # Query params query = request.GET.get('q', '') selected_facets = request.GET.getlist('facet', None) sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags') # narrow down QS with facets if selected_facets: for facet in selected_facets: sqs = sqs.narrow('tags:%s' % (facet)) # launch query if query != "": sqs = sqs.auto_query(query) uri = reverse('api_projectsheet_search', kwargs={'api_name': self.api_name, 'resource_name': self._meta.resource_name}) paginator = Paginator(request.GET, sqs, resource_uri=uri) objects = [] for result in paginator.page()['objects']: if result: bundle = self.build_bundle(obj=result.object, request=request) bundle = self.full_dehydrate(bundle) objects.append(bundle) object_list = { 'meta': paginator.page()['meta'], 'objects': objects, } self.log_throttled_access(request) return self.create_response(request, object_list)
def get_queryset(self, query_args): # Create new queryset qs = SearchQuerySet() # Are we searching all models or just a specific one (depends on # parameter set in View instantiation) if self.search_model is not None: qs = qs.models(self.search_model) # Do we have a query or are we just getting all of them? if 'q' in query_args: qs = qs.auto_query(query_args['q']) # Filter by facets if query_args['facet_filters']: # qs.filter(**query_args['facet_filters']) for facet_field in query_args['facet_filters'].keys(): qs = qs.narrow(u"{}:{}".format( facet_field, query_args['facet_filters'][facet_field]) ) # Retrieve facets and facet_counts facet_counts = [] if self.facet_fields: facets = qs for facet_field in self.facet_fields: facets = facets.facet(facet_field) facet_counts = facets.facet_counts() # Get results and return them if 'only_facets' in query_args: result = {} else: result = qs.all() return (result, facet_counts)
def all_notes(request, project_slug=None): o = {} template = 'all-notes.html' o['filtered'] = False if request.GET.get('filter'): template = 'filtered-notes.html' o['filtered'] = True qs = SearchQuerySet().models(Note) query = [] if request.GET.get('topic'): query += [ ' AND '.join([ 'related_topic_id:%s' % topic for topic in request.GET.get('topic').split(',') ]) ] if request.GET.get('project'): query += [ ' AND '.join([ 'project_id:%s' % project for project in request.GET.get('project').split(',') ]) ] qs = qs.narrow(' AND '.join(query)) if query else qs qs = qs.facet('related_topic_id').facet('project_id') facet_fields = qs.facet_counts()['fields'] topic_facets = sorted(facet_fields['related_topic_id'], key=lambda t: t[1], reverse=True) project_facets = sorted(facet_fields['project_id'], key=lambda p: p[1], reverse=True) topic_facets = [ (Topic.objects.get(id=t_id), t_count) for t_id, t_count in topic_facets[:16] ] o['topic_facets_1'] = topic_facets[:8] o['topic_facets_2'] = topic_facets[8:] if (len(topic_facets) > 8) else [] o['project_facets'] = [ (Project.objects.get(id=p_id), p_count) for p_id, p_count in project_facets ] o['notes'] = qs return render_to_response( template, o, context_instance=RequestContext(request))
def infrastructure_assets(obj): # Get all assets with valid "infrastructure" facets sqs = SearchQuerySet() sqs = sqs.facet("infrastructure") sqs = sqs.narrow("infrastructure:[* TO *]") return sqs
def facet_assets(obj, facetName): # Get all assets with valid "infrastructure" facets sqs = SearchQuerySet() sqs = sqs.facet(facetName) sqs = sqs.narrow(facetName + ":[* TO *]") return sqs
def get_context(self, request, *args, **kwargs): facet_map = ( ('building_block', (ActivityBuildingBlock, False, 10)), ('school_subject', (ActivitySchoolSubject, False, 25)), ('topic', (ActivityTopic, True, 25)), ('grade_level', (ActivityGradeLevel, False, 10)), ('age_range', (ActivityAgeRange, False, 10)), ('student_characteristics', (ActivityStudentCharacteristics, False, 10)), # noqa: E501 ('activity_type', (ActivityType, False, 10)), ('teaching_strategy', (ActivityTeachingStrategy, False, 25)), ('blooms_taxonomy_level', (ActivityBloomsTaxonomyLevel, False, 25)), # noqa: E501 ('activity_duration', (ActivityDuration, False, 10)), ('jump_start_coalition', (ActivityJumpStartCoalition, False, 25)), ('council_for_economic_education', (ActivityCouncilForEconEd, False, 25)), # noqa: E501 ) search_query = request.GET.get('q', '') # haystack cleans this string sqs = SearchQuerySet().models(ActivityPage).filter(live=True) total_activities = sqs.count() # Load selected facets selected_facets = {} facet_queries = {} for facet, facet_config in facet_map: sqs = sqs.facet(str(facet), size=facet_config[2]) if facet in request.GET and request.GET.get(facet): selected_facets[facet] = [ int(value) for value in request.GET.getlist(facet) if value.isdigit() ] facet_queries[facet] = facet + '_exact:' + ( " OR " + facet + "_exact:").join( [str(value) for value in selected_facets[facet]]) payload = { 'search_query': search_query, 'results': [], 'total_results': 0, 'total_activities': total_activities, 'selected_facets': selected_facets, 'facet_queries': facet_queries, 'all_facets': {}, } # Apply search query if it exists, but don't apply facets if search_query: sqs = sqs.filter(content=search_query).order_by( '-_score', '-date') # noqa: E501 else: sqs = sqs.order_by('-date') # Get all facets and their counts facet_counts = sqs.facet_counts() all_facets = self.get_all_facets(facet_map, sqs, facet_counts, facet_queries, selected_facets) # noqa: E501 # List all facet blocks that need to be expanded always_expanded = {'building_block', 'topic', 'school_subject'} conditionally_expanded = { facet_name for facet_name, facet_items in all_facets.items() if any(facet['selected'] is True for facet in facet_items) } expanded_facets = always_expanded.union(set(conditionally_expanded)) payload.update({ 'facet_counts': facet_counts, 'all_facets': all_facets, 'expanded_facets': expanded_facets, }) # Apply all the active facet values to our search results for facet_narrow_query in facet_queries.values(): sqs = sqs.narrow(facet_narrow_query) results = [activity.object for activity in sqs] total_results = sqs.count() payload.update({ 'results': results, 'total_results': total_results, }) self.results = payload results_per_page = validate_results_per_page(request) paginator = Paginator(payload['results'], results_per_page) current_page = validate_page_number(request, paginator) paginated_page = paginator.page(current_page) context = super(ActivityIndexPage, self).get_context(request) context.update({ 'facet_counts': facet_counts, 'facets': all_facets, 'activities': paginated_page, 'total_results': total_results, 'results_per_page': results_per_page, 'current_page': current_page, 'paginator': paginator, 'show_filters': bool(facet_queries), }) return context
class SearchQuerySetTestCase(TestCase): fixtures = ['bulk_data.json'] def setUp(self): super(SearchQuerySetTestCase, self).setUp() # Stow. self.old_unified_index = connections['default']._index self.ui = UnifiedIndex() self.bmmsi = BasicMockModelSearchIndex() self.cpkmmsi = CharPKMockModelSearchIndex() self.ui.build(indexes=[self.bmmsi, self.cpkmmsi]) connections['default']._index = self.ui # Update the "index". backend = connections['default'].get_backend() backend.clear() backend.update(self.bmmsi, MockModel.objects.all()) self.msqs = SearchQuerySet() # Stow. reset_search_queries() def tearDown(self): # Restore. connections['default']._index = self.old_unified_index super(SearchQuerySetTestCase, self).tearDown() def test_len(self): self.assertEqual(len(self.msqs), 23) def test_repr(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertRegexpMatches( repr(self.msqs), r'^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object' r' at 0x[0-9A-Fa-f]+>, using=None>$') def test_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) msqs = self.msqs.all() results = [int(res.pk) for res in msqs] self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]]) self.assertEqual(len(connections['default'].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.msqs.all() self.assertEqual([int(res.pk) for res in results[1:11]], [res.pk for res in MOCK_SEARCH_RESULTS[1:11]]) self.assertEqual(len(connections['default'].queries), 1) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.msqs.all() self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk) self.assertEqual(len(connections['default'].queries), 1) def test_manual_iter(self): results = self.msqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) check = [result.pk for result in results._manual_iter()] self.assertEqual(check, [ u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'10', u'11', u'12', u'13', u'14', u'15', u'16', u'17', u'18', u'19', u'20', u'21', u'22', u'23' ]) self.assertEqual(len(connections['default'].queries), 3) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) # Test to ensure we properly fill the cache, even if we get fewer # results back (not a handled model) than the hit count indicates. # This will hang indefinitely if broken. old_ui = self.ui self.ui.build(indexes=[self.cpkmmsi]) connections['default']._index = self.ui self.cpkmmsi.update() results = self.msqs.all() loaded = [result.pk for result in results._manual_iter()] self.assertEqual(loaded, [u'sometext', u'1234']) self.assertEqual(len(connections['default'].queries), 1) connections['default']._index = old_ui def test_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.msqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 20) self.assertEqual(len(connections['default'].queries), 2) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) # Test to ensure we properly fill the cache, even if we get fewer # results back (not a handled model) than the hit count indicates. sqs = SearchQuerySet().all() sqs.query.backend = MixedMockSearchBackend('default') results = sqs self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 0) self.assertEqual([ int(result.pk) for result in results._result_cache if result is not None ], []) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 9) self.assertEqual([ int(result.pk) for result in results._result_cache if result is not None ], [1, 2, 3, 4, 5, 6, 7, 8, 10]) self.assertEqual(len(connections['default'].queries), 2) results._fill_cache(10, 20) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 17) self.assertEqual([ int(result.pk) for result in results._result_cache if result is not None ], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20]) self.assertEqual(len(connections['default'].queries), 4) results._fill_cache(20, 30) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 20) self.assertEqual([ int(result.pk) for result in results._result_cache if result is not None ], [ 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23 ]) self.assertEqual(len(connections['default'].queries), 6) def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.msqs._cache_is_full(), False) results = self.msqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 3) def test_all(self): sqs = self.msqs.all() self.assertTrue(isinstance(sqs, SearchQuerySet)) def test_filter(self): sqs = self.msqs.filter(content='foo') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_exclude(self): sqs = self.msqs.exclude(content='foo') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_order_by(self): sqs = self.msqs.order_by('foo') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue('foo' in sqs.query.order_by) def test_models(self): # Stow. old_unified_index = connections['default']._index ui = UnifiedIndex() bmmsi = BasicMockModelSearchIndex() bammsi = BasicAnotherMockModelSearchIndex() ui.build(indexes=[bmmsi, bammsi]) connections['default']._index = ui msqs = SearchQuerySet() sqs = msqs.all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 0) sqs = msqs.models(MockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 1) sqs = msqs.models(MockModel, AnotherMockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 2) # This will produce a warning. ui.build(indexes=[bmmsi]) sqs = msqs.models(AnotherMockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 1) def test_result_class(self): sqs = self.msqs.all() self.assertTrue(issubclass(sqs.query.result_class, SearchResult)) # Custom class. class IttyBittyResult(object): pass sqs = self.msqs.result_class(IttyBittyResult) self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult)) # Reset to default. sqs = self.msqs.result_class(None) self.assertTrue(issubclass(sqs.query.result_class, SearchResult)) def test_boost(self): sqs = self.msqs.boost('foo', 10) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.boost.keys()), 1) def test_highlight(self): sqs = self.msqs.highlight() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.query.highlight, True) def test_spelling(self): # Test the case where spelling support is disabled. sqs = self.msqs.filter(content='Indx') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.spelling_suggestion(), None) self.assertEqual(sqs.spelling_suggestion('indexy'), None) def test_raw_search(self): self.assertEqual(len(self.msqs.raw_search('foo')), 23) self.assertEqual( len( self.msqs.raw_search( '(content__exact:hello AND content__exact:world)')), 23) def test_load_all(self): # Models with character primary keys. sqs = SearchQuerySet() sqs.query.backend = CharPKMockSearchBackend('charpk') results = sqs.load_all().all() self.assertEqual(len(results._result_cache), 0) results._fill_cache(0, 2) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 2) # If nothing is handled, you get nothing. old_ui = connections['default']._index ui = UnifiedIndex() ui.build(indexes=[]) connections['default']._index = ui sqs = self.msqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs), 0) connections['default']._index = old_ui # For full tests, see the solr_backend. def test_load_all_read_queryset(self): # Stow. old_ui = connections['default']._index ui = UnifiedIndex() gafmmsi = GhettoAFifthMockModelSearchIndex() ui.build(indexes=[gafmmsi]) connections['default']._index = ui gafmmsi.update() sqs = SearchQuerySet() results = sqs.load_all().all() results.query.backend = ReadQuerySetMockSearchBackend('default') results._fill_cache(0, 2) # The deleted result isn't returned self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 1) # Register a SearchIndex with a read_queryset that returns deleted items rqstsi = TextReadQuerySetTestSearchIndex() ui.build(indexes=[rqstsi]) rqstsi.update() sqs = SearchQuerySet() results = sqs.load_all().all() results.query.backend = ReadQuerySetMockSearchBackend('default') results._fill_cache(0, 2) # Both the deleted and not deleted items are returned self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 2) # Restore. connections['default']._index = old_ui def test_auto_query(self): sqs = self.msqs.auto_query('test search -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains=test search -stuff>') sqs = self.msqs.auto_query('test "my thing" search -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__contains=test "my thing" search -stuff>') sqs = self.msqs.auto_query( 'test "my thing" search \'moar quotes\' -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__contains=test "my thing" search \'moar quotes\' -stuff>' ) sqs = self.msqs.auto_query( 'test "my thing" search \'moar quotes\' "foo -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__contains=test "my thing" search \'moar quotes\' "foo -stuff>' ) sqs = self.msqs.auto_query('test - stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__contains=test - stuff>") # Ensure bits in exact matches get escaped properly as well. sqs = self.msqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">') # Now with a different fieldname sqs = self.msqs.auto_query('test search -stuff', fieldname='title') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND title__contains=test search -stuff>") sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname='title') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND title__contains=test "my thing" search -stuff>') def test_count(self): self.assertEqual(self.msqs.count(), 23) def test_facet_counts(self): self.assertEqual(self.msqs.facet_counts(), {}) def test_best_match(self): self.assertTrue(isinstance(self.msqs.best_match(), SearchResult)) def test_latest(self): self.assertTrue(isinstance(self.msqs.latest('pub_date'), SearchResult)) def test_more_like_this(self): mock = MockModel() mock.id = 1 self.assertEqual(len(self.msqs.more_like_this(mock)), 23) def test_facets(self): sqs = self.msqs.facet('foo') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.facets), 1) sqs2 = self.msqs.facet('foo').facet('bar') self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.facets), 2) def test_date_facets(self): try: sqs = self.msqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='smarblaph') self.fail() except FacetingError as e: self.assertEqual( str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second." ) sqs = self.msqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.date_facets), 1) sqs2 = self.msqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month').date_facet( 'bar', start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='year') self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.date_facets), 2) def test_query_facets(self): sqs = self.msqs.query_facet('foo', '[bar TO *]') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_facets), 1) sqs2 = self.msqs.query_facet('foo', '[bar TO *]').query_facet( 'bar', '[100 TO 499]') self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.query_facets), 2) # Test multiple query facets on a single field sqs3 = self.msqs.query_facet('foo', '[bar TO *]').query_facet( 'bar', '[100 TO 499]').query_facet('foo', '[1000 TO 1499]') self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.query_facets), 3) def test_stats(self): sqs = self.msqs.stats_facet('foo', 'bar') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.stats), 1) sqs2 = self.msqs.stats_facet('foo', 'bar').stats_facet('foo', 'baz') self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.stats), 1) sqs3 = self.msqs.stats_facet('foo', 'bar').stats_facet('moof', 'baz') self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.stats), 2) def test_narrow(self): sqs = self.msqs.narrow('foo:moof') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.narrow_queries), 1) def test_clone(self): results = self.msqs.filter(foo='bar', foo__lt='10') clone = results._clone() self.assertTrue(isinstance(clone, SearchQuerySet)) self.assertEqual(str(clone.query), str(results.query)) self.assertEqual(clone._result_cache, []) self.assertEqual(clone._result_count, None) self.assertEqual(clone._cache_full, False) self.assertEqual(clone._using, results._using) def test_using(self): sqs = SearchQuerySet(using='default') self.assertNotEqual(sqs.query, None) self.assertEqual(sqs.query._using, 'default') def test_chaining(self): sqs = self.msqs.filter(content='foo') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) # A second instance should inherit none of the changes from above. sqs = self.msqs.filter(content='bar') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_none(self): sqs = self.msqs.none() self.assertTrue(isinstance(sqs, EmptySearchQuerySet)) self.assertEqual(len(sqs), 0) def test___and__(self): sqs1 = self.msqs.filter(content='foo') sqs2 = self.msqs.filter(content='bar') sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) def test___or__(self): sqs1 = self.msqs.filter(content='foo') sqs2 = self.msqs.filter(content='bar') sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) def test_and_or(self): """ Combining AND queries with OR should give AND(OR(a, b), OR(c, d)) """ sqs1 = self.msqs.filter(content='foo').filter(content='oof') sqs2 = self.msqs.filter(content='bar').filter(content='rab') sqs = sqs1 | sqs2 self.assertEqual(sqs.query.query_filter.connector, 'OR') self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)) self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter)) def test_or_and(self): """ Combining OR queries with AND should give OR(AND(a, b), AND(c, d)) """ sqs1 = self.msqs.filter(content='foo').filter_or(content='oof') sqs2 = self.msqs.filter(content='bar').filter_or(content='rab') sqs = sqs1 & sqs2 self.assertEqual(sqs.query.query_filter.connector, 'AND') self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)) self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter))
def haystack_search_api(request, format="json", **kwargs): """ View that drives the search api """ from haystack.inputs import Raw from haystack.query import SearchQuerySet, SQ # Retrieve Query Params id = request.REQUEST.get("id", None) query = request.REQUEST.get('q', None) category = request.REQUEST.get("category", None) limit = int( request.REQUEST.get( "limit", getattr(settings, "HAYSTACK_SEARCH_RESULTS_PER_PAGE", 20))) startIndex = int(request.REQUEST.get("startIndex", 0)) sort = request.REQUEST.get("sort", "relevance") type_facets = request.REQUEST.get("type", None) format = request.REQUEST.get("format", format) date_start = request.REQUEST.get("start_date", None) date_end = request.REQUEST.get("end_date", None) keyword = request.REQUEST.get("kw", None) service = request.REQUEST.get("service", None) local = request.REQUEST.get("local", None) # Geospatial Elements bbox = request.REQUEST.get("extent", None) ts = time() sqs = SearchQuerySet() limit = min(limit, 500) # Filter by ID if id: sqs = sqs.narrow("django_id:%s" % id) # Filter by Type and subtype if type_facets is not None: type_facets = type_facets.replace("owner", "user").split(",") subtype_facets = ["vector", "raster"] types = [] subtypes = [] for type in type_facets: if type in ["map", "layer", "user", "document", "group"]: # Type is one of our Major Types (not a sub type) types.append(type) elif type in subtype_facets: subtypes.append(type) if len(subtypes) > 0: for sub_type in subtype_facets: if sub_type not in subtypes: sqs = sqs.exclude(subtype='%s' % sub_type) if len(types) > 0: sqs = sqs.narrow("type:%s" % ','.join(map(str, types))) # Filter by Query Params # haystack bug? if boosted fields aren't included in the # query, then the score won't be affected by the boost if query: if query.startswith('"') or query.startswith('\''): #Match exact phrase phrase = query.replace('"', '') sqs = sqs.filter( SQ(title__exact=phrase) | SQ(abstract__exact=phrase) | SQ(content__exact=phrase)) else: words = query.split() for word in range(0, len(words)): if word == 0: sqs = sqs.filter( SQ(title=Raw(words[word])) | SQ(abstract=Raw(words[word])) | SQ(content=Raw(words[word]))) elif words[word] in ["AND", "OR"]: pass elif words[word - 1] == "OR": #previous word OR this word sqs = sqs.filter_or( SQ(title=Raw(words[word])) | SQ(abstract=Raw(words[word])) | SQ(content=Raw(words[word]))) else: #previous word AND this word sqs = sqs.filter( SQ(title=Raw(words[word])) | SQ(abstract=Raw(words[word])) | SQ(content=Raw(words[word]))) # filter by cateory if category: sqs = sqs.narrow('category:%s' % category) #filter by keyword if keyword: sqs = sqs.narrow('keywords:%s' % keyword) if date_start: sqs = sqs.filter(SQ(date__gte=date_start)) if date_end: sqs = sqs.filter(SQ(date__lte=date_end)) """ ### Code to filter on temporal extent start/end dates instead if date_start or date_end: #Exclude results with no dates at all sqs = sqs.filter( SQ(temporal_extent_start=Raw("[* TO *]")) | SQ(temporal_extent_end=Raw("[* TO *]")) ) if temporal_start and temporal_end: #Return anything with a start date < date_end or an end date > date_start sqs = sqs.filter( SQ(temporal_extent_end__gte=date_start) | SQ(temporal_extent_start__lte=date_end) ) elif temporal_start: #Exclude anything with an end date <date_start sqs = sqs.exclude( SQ(temporal_extent_end__lte=date_start) ) elif temporal_end: #Exclude anything with a start date > date_end sqs = sqs.exclude( SQ(temporal_extent_start__gte=date_end) ) """ if bbox: left, right, bottom, top = bbox.split(',') sqs = sqs.filter( # first check if the bbox has at least one point inside the window SQ(bbox_left__gte=left) & SQ(bbox_left__lte=right) & SQ(bbox_top__gte=bottom) & SQ(bbox_top__lte=top) | #check top_left is inside the window SQ(bbox_right__lte=right) & SQ(bbox_right__gte=left) & SQ(bbox_top__lte=top) & SQ(bbox_top__gte=bottom) | #check top_right is inside the window SQ(bbox_bottom__gte=bottom) & SQ(bbox_bottom__lte=top) & SQ(bbox_right__lte=right) & SQ(bbox_right__gte=left) | #check bottom_right is inside the window SQ(bbox_top__lte=top) & SQ(bbox_top__gte=bottom) & SQ(bbox_left__gte=left) & SQ(bbox_left__lte=right) | #check bottom_left is inside the window # then check if the bbox is including the window SQ(bbox_left__lte=left) & SQ(bbox_right__gte=right) & SQ(bbox_bottom__lte=bottom) & SQ(bbox_top__gte=top)) # Filter by permissions ''' ### Takes too long with many results. ### Instead, show all results but disable links on restricted ones. for i, result in enumerate(sqs): if result.type == 'layer': if not request.user.has_perm('layers.view_layer',obj = result.object): sqs = sqs.exclude(id = result.id) if result.type == 'map': if not request.user.has_perm('maps.view_map',obj = result.object): sqs = sqs.exclude(id = result.id) ''' #filter by service ''' if service: sqs = sqs.narrow('service:%s' % service) if local: sqs = sqs.narrow('local:%s' % local) ''' # Apply Sort # TODO: Handle for Revised sort types # [relevance, alphabetically, rating, created, updated, popularity] if sort.lower() == "newest": sqs = sqs.order_by("-modified") elif sort.lower() == "oldest": sqs = sqs.order_by("modified") elif sort.lower() == "alphaaz": sqs = sqs.order_by("title_sortable") elif sort.lower() == "alphaza": sqs = sqs.order_by("-title_sortable") elif sort.lower() == "popularity": sqs = sqs.order_by("-popular_count") else: sqs = sqs.order_by("-_score") # Setup Search Results results = [] items = [] # Build the result based on the limit for i, result in enumerate(sqs[startIndex:startIndex + limit]): logger.info(result) data = result.get_stored_fields() resource = None if "modified" in data: data["modified"] = data["modified"].strftime( "%Y-%m-%dT%H:%M:%S.%f") if "temporal_extent_start" in data and data[ "temporal_extent_start"] is not None: data["temporal_extent_start"] = data[ "temporal_extent_start"].strftime("%Y-%m-%dT%H:%M:%S.%f") if "temporal_extent_end" in data and data[ "temporal_extent_end"] is not None: data["temporal_extent_end"] = data["temporal_extent_end"].strftime( "%Y-%m-%dT%H:%M:%S.%f") if data['type'] == "map": resource = MapNormalizer(Map.objects.get(pk=data['oid'])) elif data['type'] == "layer": resource = LayerNormalizer(Layer.objects.get(pk=data['oid'])) elif data['type'] == "user": resource = OwnerNormalizer(Profile.objects.get(pk=data['oid'])) elif data['type'] == "document": resource = DocumentNormalizer(Document.objects.get(pk=data['oid'])) elif data[ 'type'] == "group" and "geonode.contrib.groups" in settings.INSTALLED_APPS: resource = GroupNormalizer(Group.objects.get(pk=data['oid'])) if resource: resource.rating = data["rating"] if "rating" in data else 0 results.append(data) items.append(resource) # Setup Facet Counts sqs = sqs.facet("type").facet("subtype") sqs = sqs.facet('category') sqs = sqs.facet('keywords') sqs = sqs.facet('service') sqs = sqs.facet('local') facet_counts = sqs.facet_counts() # Prepare Search Results data = { "success": True, "total": sqs.count(), "query_info": { "q": query, "startIndex": startIndex, "limit": limit, "sort": sort, "type": type_facets, }, "results": results, "facets": dict( facet_counts.get("fields")['type'] + facet_counts.get('fields')['subtype']) if sqs.count() > 0 else [], "categories": { facet[0]: facet[1] for facet in facet_counts.get('fields')['category'] } if sqs.count() > 0 else {}, "keywords": { facet[0]: facet[1] for facet in facet_counts.get('fields')['keywords'] } if sqs.count() > 0 else {}, } # Return Results ts1 = time() - ts if format == "html": #Send to search/explore page return data, items elif format == "raw": return HttpResponse(json.dumps(data), mimetype="application/json") else: query = query_from_request(request, kwargs) return _search_json(query, items, data["facets"], ts1)
def get_queryset(self, *args, **kwargs): qs = SearchQuerySet().all() qs = qs.facet('categories') if self.request.user.is_authenticated(): qs = qs.facet('groups') if self.request.query_params.get('q'): #qs = qs.filter_and(title__contains=self.request.query_params.get('q')) qs = qs.auto_query(self.request.query_params.get('q')) distance = None try: for k, v in self.request.query_params.items(): if k in D.UNITS.keys(): distance = {k: v} except Exception as e: logging.error(e) point = None try: point = Point(float(self.request.query_params['lng']), float(self.request.query_params['lat'])) except Exception as e: logging.error(e) if distance and point: qs = qs or SearchQuerySet() qs = qs.dwithin('location', point, D(**distance)).distance('location', point) try: if self.request.query_params.get('w') and self.request.query_params.get('s')\ and self.request.query_params.get('n') and self.request.query_params.get('e'): bottom_left = Point(float(self.request.query_params['w']), float(self.request.query_params['s'])) top_right = Point(float(self.request.query_params['e']), float(self.request.query_params['n'])) qs = qs.within('location', bottom_left, top_right) except: pass param_facet_url = self.request.query_params.get('selected_facets', '').split(',') flag_group = False for param_facet in param_facet_url: if ":" not in param_facet: continue field, value = param_facet.split(":", 1) if value and field.split('_')[1] == 'exact': try: # Validate that the user belongs to the group if field.split('_')[0] == 'groups': InterestGroup.objects.\ filter(Q(memberships__user=self.request.user) | Q(owner=self.request.user)).\ get(slug=value) flag_group = True qs = qs.narrow('%s:"%s"' % (field, qs.query.clean(value))) except InterestGroup.DoesNotExist: # The user hasnt permissiont to filter by this group. pass self.facets = [] group_facet = get_facet_by_name(param_facet_url, qs.facet_counts()['fields'].items(), 'groups', self.request.user) if group_facet: self.facets.append(group_facet) order_by = self.request.query_params.get('order_by') if order_by: # TODO: Order by distance not found if order_by is 'distance': if distance and point: qs = qs.order_by(order_by) else: qs = qs.order_by(order_by) if not flag_group: qs = qs.narrow('%s:"%s"' % ('groups_exact', qs.query.clean('public'))) self.facets.append( get_facet_by_name(param_facet_url, qs.facet_counts()['fields'].items(), 'categories')) return qs
def build_form(self, *args, **kwargs): sqs = SearchQuerySet() sqs = sqs.narrow('instance:"%s"' % self.request.instance.label) self.searchqueryset = sqs return super(InstanceSearchView, self).build_form(*args, **kwargs)
class SearchQuerySetTestCase(TestCase): fixtures = ["base_data.json", "bulk_data.json"] def setUp(self): super(SearchQuerySetTestCase, self).setUp() # Stow. self.old_unified_index = connections["default"]._index self.ui = UnifiedIndex() self.bmmsi = BasicMockModelSearchIndex() self.cpkmmsi = CharPKMockModelSearchIndex() self.uuidmmsi = SimpleMockUUIDModelIndex() self.ui.build(indexes=[self.bmmsi, self.cpkmmsi, self.uuidmmsi]) connections["default"]._index = self.ui # Update the "index". backend = connections["default"].get_backend() backend.clear() backend.update(self.bmmsi, MockModel.objects.all()) self.msqs = SearchQuerySet() # Stow. reset_search_queries() def tearDown(self): # Restore. connections["default"]._index = self.old_unified_index super(SearchQuerySetTestCase, self).tearDown() def test_len(self): self.assertEqual(len(self.msqs), 23) def test_repr(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) self.assertRegexpMatches( repr(self.msqs), r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object" r" at 0x[0-9A-Fa-f]+>, using=None>$", ) def test_iter(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) msqs = self.msqs.all() results = [int(res.pk) for res in iter(msqs)] self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]]) self.assertEqual(len(connections["default"].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) results = self.msqs.all() self.assertEqual( [int(res.pk) for res in results[1:11]], [res.pk for res in MOCK_SEARCH_RESULTS[1:11]], ) self.assertEqual(len(connections["default"].queries), 1) reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) results = self.msqs.all() self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk) self.assertEqual(len(connections["default"].queries), 1) def test_manual_iter(self): results = self.msqs.all() reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) check = [result.pk for result in results._manual_iter()] self.assertEqual( check, [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", ], ) self.assertEqual(len(connections["default"].queries), 3) reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) # Test to ensure we properly fill the cache, even if we get fewer # results back (not a handled model) than the hit count indicates. # This will hang indefinitely if broken. # CharPK testing old_ui = self.ui self.ui.build(indexes=[self.cpkmmsi]) connections["default"]._index = self.ui self.cpkmmsi.update() results = self.msqs.all() loaded = [result.pk for result in results._manual_iter()] self.assertEqual(loaded, ["sometext", "1234"]) self.assertEqual(len(connections["default"].queries), 1) # UUID testing self.ui.build(indexes=[self.uuidmmsi]) connections["default"]._index = self.ui self.uuidmmsi.update() results = self.msqs.all() loaded = [result.pk for result in results._manual_iter()] self.assertEqual( loaded, [ "53554c58-7051-4350-bcc9-dad75eb248a9", "77554c58-7051-4350-bcc9-dad75eb24888", ], ) connections["default"]._index = old_ui def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) self.assertEqual(self.msqs._cache_is_full(), False) results = self.msqs.all() fire_the_iterator_and_fill_cache = list(results) self.assertEqual(23, len(fire_the_iterator_and_fill_cache)) self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections["default"].queries), 4) def test_all(self): sqs = self.msqs.all() self.assertTrue(isinstance(sqs, SearchQuerySet)) def test_filter(self): sqs = self.msqs.filter(content="foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_exclude(self): sqs = self.msqs.exclude(content="foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_order_by(self): sqs = self.msqs.order_by("foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue("foo" in sqs.query.order_by) def test_models(self): # Stow. old_unified_index = connections["default"]._index ui = UnifiedIndex() bmmsi = BasicMockModelSearchIndex() bammsi = BasicAnotherMockModelSearchIndex() ui.build(indexes=[bmmsi, bammsi]) connections["default"]._index = ui msqs = SearchQuerySet() sqs = msqs.all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 0) sqs = msqs.models(MockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 1) sqs = msqs.models(MockModel, AnotherMockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 2) # This will produce a warning. ui.build(indexes=[bmmsi]) sqs = msqs.models(AnotherMockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 1) def test_result_class(self): sqs = self.msqs.all() self.assertTrue(issubclass(sqs.query.result_class, SearchResult)) # Custom class. class IttyBittyResult(object): pass sqs = self.msqs.result_class(IttyBittyResult) self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult)) # Reset to default. sqs = self.msqs.result_class(None) self.assertTrue(issubclass(sqs.query.result_class, SearchResult)) def test_boost(self): sqs = self.msqs.boost("foo", 10) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.boost.keys()), 1) def test_highlight(self): sqs = self.msqs.highlight() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.query.highlight, True) def test_spelling_override(self): sqs = self.msqs.filter(content="not the spellchecking query") self.assertEqual(sqs.query.spelling_query, None) sqs = self.msqs.set_spelling_query("override") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.query.spelling_query, "override") def test_spelling_suggestions(self): # Test the case where spelling support is disabled. sqs = self.msqs.filter(content="Indx") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.spelling_suggestion(), None) self.assertEqual(sqs.spelling_suggestion("indexy"), None) def test_raw_search(self): self.assertEqual(len(self.msqs.raw_search("foo")), 23) self.assertEqual( len( self.msqs.raw_search("(content__exact:hello AND content__exact:world)") ), 23, ) def test_load_all(self): # Models with character primary keys. sqs = SearchQuerySet() sqs.query.backend = CharPKMockSearchBackend("charpk") results = sqs.load_all().all() self.assertEqual(len(results._result_cache), 0) results._fill_cache(0, 2) self.assertEqual( len([result for result in results._result_cache if result is not None]), 2 ) # Models with uuid primary keys. sqs = SearchQuerySet() sqs.query.backend = UUIDMockSearchBackend("uuid") results = sqs.load_all().all() self.assertEqual(len(results._result_cache), 0) results._fill_cache(0, 2) self.assertEqual( len([result for result in results._result_cache if result is not None]), 2 ) # If nothing is handled, you get nothing. old_ui = connections["default"]._index ui = UnifiedIndex() ui.build(indexes=[]) connections["default"]._index = ui sqs = self.msqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs), 0) connections["default"]._index = old_ui # For full tests, see the solr_backend. def test_load_all_read_queryset(self): # Stow. old_ui = connections["default"]._index ui = UnifiedIndex() gafmmsi = GhettoAFifthMockModelSearchIndex() ui.build(indexes=[gafmmsi]) connections["default"]._index = ui gafmmsi.update() sqs = SearchQuerySet() results = sqs.load_all().all() results.query.backend = ReadQuerySetMockSearchBackend("default") results._fill_cache(0, 2) # The deleted result isn't returned self.assertEqual( len([result for result in results._result_cache if result is not None]), 1 ) # Register a SearchIndex with a read_queryset that returns deleted items rqstsi = TextReadQuerySetTestSearchIndex() ui.build(indexes=[rqstsi]) rqstsi.update() sqs = SearchQuerySet() results = sqs.load_all().all() results.query.backend = ReadQuerySetMockSearchBackend("default") results._fill_cache(0, 2) # Both the deleted and not deleted items are returned self.assertEqual( len([result for result in results._result_cache if result is not None]), 2 ) # Restore. connections["default"]._index = old_ui def test_auto_query(self): sqs = self.msqs.auto_query("test search -stuff") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), "<SQ: AND content__content=test search -stuff>", ) sqs = self.msqs.auto_query('test "my thing" search -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__content=test "my thing" search -stuff>', ) sqs = self.msqs.auto_query("test \"my thing\" search 'moar quotes' -stuff") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), "<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>", ) sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>', ) sqs = self.msqs.auto_query("test - stuff") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), "<SQ: AND content__content=test - stuff>" ) # Ensure bits in exact matches get escaped properly as well. sqs = self.msqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__content="pants:rule">' ) # Now with a different fieldname sqs = self.msqs.auto_query("test search -stuff", fieldname="title") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), "<SQ: AND title__content=test search -stuff>" ) sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname="title") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND title__content=test "my thing" search -stuff>', ) def test_count(self): self.assertEqual(self.msqs.count(), 23) def test_facet_counts(self): self.assertEqual(self.msqs.facet_counts(), {}) def test_best_match(self): self.assertTrue(isinstance(self.msqs.best_match(), SearchResult)) def test_latest(self): self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult)) def test_more_like_this(self): mock = MockModel() mock.id = 1 self.assertEqual(len(self.msqs.more_like_this(mock)), 23) def test_facets(self): sqs = self.msqs.facet("foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.facets), 1) sqs2 = self.msqs.facet("foo").facet("bar") self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.facets), 2) def test_date_facets(self): try: sqs = self.msqs.date_facet( "foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="smarblaph", ) self.fail() except FacetingError as e: self.assertEqual( str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.", ) sqs = self.msqs.date_facet( "foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month", ) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.date_facets), 1) sqs2 = self.msqs.date_facet( "foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month", ).date_facet( "bar", start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="year", ) self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.date_facets), 2) def test_query_facets(self): sqs = self.msqs.query_facet("foo", "[bar TO *]") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_facets), 1) sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet( "bar", "[100 TO 499]" ) self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.query_facets), 2) # Test multiple query facets on a single field sqs3 = ( self.msqs.query_facet("foo", "[bar TO *]") .query_facet("bar", "[100 TO 499]") .query_facet("foo", "[1000 TO 1499]") ) self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.query_facets), 3) def test_stats(self): sqs = self.msqs.stats_facet("foo", "bar") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.stats), 1) sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz") self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.stats), 1) sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz") self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.stats), 2) def test_narrow(self): sqs = self.msqs.narrow("foo:moof") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.narrow_queries), 1) def test_clone(self): results = self.msqs.filter(foo="bar", foo__lt="10") clone = results._clone() self.assertTrue(isinstance(clone, SearchQuerySet)) self.assertEqual(str(clone.query), str(results.query)) self.assertEqual(clone._result_cache, []) self.assertEqual(clone._result_count, None) self.assertEqual(clone._cache_full, False) self.assertEqual(clone._using, results._using) def test_using(self): sqs = SearchQuerySet(using="default") self.assertNotEqual(sqs.query, None) self.assertEqual(sqs.query._using, "default") def test_chaining(self): sqs = self.msqs.filter(content="foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) # A second instance should inherit none of the changes from above. sqs = self.msqs.filter(content="bar") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_none(self): sqs = self.msqs.none() self.assertTrue(isinstance(sqs, EmptySearchQuerySet)) self.assertEqual(len(sqs), 0) def test___and__(self): sqs1 = self.msqs.filter(content="foo") sqs2 = self.msqs.filter(content="bar") sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) def test___or__(self): sqs1 = self.msqs.filter(content="foo") sqs2 = self.msqs.filter(content="bar") sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) def test_and_or(self): """ Combining AND queries with OR should give AND(OR(a, b), OR(c, d)) """ sqs1 = self.msqs.filter(content="foo").filter(content="oof") sqs2 = self.msqs.filter(content="bar").filter(content="rab") sqs = sqs1 | sqs2 self.assertEqual(sqs.query.query_filter.connector, "OR") self.assertEqual( repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter) ) self.assertEqual( repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter) ) def test_or_and(self): """ Combining OR queries with AND should give OR(AND(a, b), AND(c, d)) """ sqs1 = self.msqs.filter(content="foo").filter_or(content="oof") sqs2 = self.msqs.filter(content="bar").filter_or(content="rab") sqs = sqs1 & sqs2 self.assertEqual(sqs.query.query_filter.connector, "AND") self.assertEqual( repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter) ) self.assertEqual( repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter) )
class SearchQuerySetTestCase(TestCase): fixtures = ["base_data.json", "bulk_data.json"] def setUp(self): super(SearchQuerySetTestCase, self).setUp() # Stow. self.old_unified_index = connections["default"]._index self.ui = UnifiedIndex() self.bmmsi = BasicMockModelSearchIndex() self.cpkmmsi = CharPKMockModelSearchIndex() self.ui.build(indexes=[self.bmmsi, self.cpkmmsi]) connections["default"]._index = self.ui # Update the "index". backend = connections["default"].get_backend() backend.clear() backend.update(self.bmmsi, MockModel.objects.all()) self.msqs = SearchQuerySet() # Stow. reset_search_queries() def tearDown(self): # Restore. connections["default"]._index = self.old_unified_index super(SearchQuerySetTestCase, self).tearDown() def test_len(self): self.assertEqual(len(self.msqs), 23) def test_repr(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) self.assertRegexpMatches( repr(self.msqs), r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object" r" at 0x[0-9A-Fa-f]+>, using=None>$", ) def test_iter(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) msqs = self.msqs.all() results = [int(res.pk) for res in iter(msqs)] self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]]) self.assertEqual(len(connections["default"].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) results = self.msqs.all() self.assertEqual([int(res.pk) for res in results[1:11]], [res.pk for res in MOCK_SEARCH_RESULTS[1:11]]) self.assertEqual(len(connections["default"].queries), 1) reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) results = self.msqs.all() self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk) self.assertEqual(len(connections["default"].queries), 1) def test_manual_iter(self): results = self.msqs.all() reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) check = [result.pk for result in results._manual_iter()] self.assertEqual( check, [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20", "21", "22", "23", ], ) self.assertEqual(len(connections["default"].queries), 3) reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) # Test to ensure we properly fill the cache, even if we get fewer # results back (not a handled model) than the hit count indicates. # This will hang indefinitely if broken. old_ui = self.ui self.ui.build(indexes=[self.cpkmmsi]) connections["default"]._index = self.ui self.cpkmmsi.update() results = self.msqs.all() loaded = [result.pk for result in results._manual_iter()] self.assertEqual(loaded, ["sometext", "1234"]) self.assertEqual(len(connections["default"].queries), 1) connections["default"]._index = old_ui def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections["default"].queries), 0) self.assertEqual(self.msqs._cache_is_full(), False) results = self.msqs.all() fire_the_iterator_and_fill_cache = list(results) self.assertEqual(23, len(fire_the_iterator_and_fill_cache)) self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections["default"].queries), 4) def test_all(self): sqs = self.msqs.all() self.assertTrue(isinstance(sqs, SearchQuerySet)) def test_filter(self): sqs = self.msqs.filter(content="foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_exclude(self): sqs = self.msqs.exclude(content="foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_order_by(self): sqs = self.msqs.order_by("foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue("foo" in sqs.query.order_by) def test_models(self): # Stow. old_unified_index = connections["default"]._index ui = UnifiedIndex() bmmsi = BasicMockModelSearchIndex() bammsi = BasicAnotherMockModelSearchIndex() ui.build(indexes=[bmmsi, bammsi]) connections["default"]._index = ui msqs = SearchQuerySet() sqs = msqs.all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 0) sqs = msqs.models(MockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 1) sqs = msqs.models(MockModel, AnotherMockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 2) # This will produce a warning. ui.build(indexes=[bmmsi]) sqs = msqs.models(AnotherMockModel) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.models), 1) def test_result_class(self): sqs = self.msqs.all() self.assertTrue(issubclass(sqs.query.result_class, SearchResult)) # Custom class. class IttyBittyResult(object): pass sqs = self.msqs.result_class(IttyBittyResult) self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult)) # Reset to default. sqs = self.msqs.result_class(None) self.assertTrue(issubclass(sqs.query.result_class, SearchResult)) def test_boost(self): sqs = self.msqs.boost("foo", 10) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.boost.keys()), 1) def test_highlight(self): sqs = self.msqs.highlight() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.query.highlight, True) def test_spelling_override(self): sqs = self.msqs.filter(content="not the spellchecking query") self.assertEqual(sqs.query.spelling_query, None) sqs = self.msqs.set_spelling_query("override") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.query.spelling_query, "override") def test_spelling_suggestions(self): # Test the case where spelling support is disabled. sqs = self.msqs.filter(content="Indx") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(sqs.spelling_suggestion(), None) self.assertEqual(sqs.spelling_suggestion("indexy"), None) def test_raw_search(self): self.assertEqual(len(self.msqs.raw_search("foo")), 23) self.assertEqual(len(self.msqs.raw_search("(content__exact:hello AND content__exact:world)")), 23) def test_load_all(self): # Models with character primary keys. sqs = SearchQuerySet() sqs.query.backend = CharPKMockSearchBackend("charpk") results = sqs.load_all().all() self.assertEqual(len(results._result_cache), 0) results._fill_cache(0, 2) self.assertEqual(len([result for result in results._result_cache if result is not None]), 2) # If nothing is handled, you get nothing. old_ui = connections["default"]._index ui = UnifiedIndex() ui.build(indexes=[]) connections["default"]._index = ui sqs = self.msqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs), 0) connections["default"]._index = old_ui # For full tests, see the solr_backend. def test_load_all_read_queryset(self): # Stow. old_ui = connections["default"]._index ui = UnifiedIndex() gafmmsi = GhettoAFifthMockModelSearchIndex() ui.build(indexes=[gafmmsi]) connections["default"]._index = ui gafmmsi.update() sqs = SearchQuerySet() results = sqs.load_all().all() results.query.backend = ReadQuerySetMockSearchBackend("default") results._fill_cache(0, 2) # The deleted result isn't returned self.assertEqual(len([result for result in results._result_cache if result is not None]), 1) # Register a SearchIndex with a read_queryset that returns deleted items rqstsi = TextReadQuerySetTestSearchIndex() ui.build(indexes=[rqstsi]) rqstsi.update() sqs = SearchQuerySet() results = sqs.load_all().all() results.query.backend = ReadQuerySetMockSearchBackend("default") results._fill_cache(0, 2) # Both the deleted and not deleted items are returned self.assertEqual(len([result for result in results._result_cache if result is not None]), 2) # Restore. connections["default"]._index = old_ui def test_auto_query(self): sqs = self.msqs.auto_query("test search -stuff") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__content=test search -stuff>") sqs = self.msqs.auto_query('test "my thing" search -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__content=test "my thing" search -stuff>') sqs = self.msqs.auto_query("test \"my thing\" search 'moar quotes' -stuff") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), "<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>" ) sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual( repr(sqs.query.query_filter), '<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>', ) sqs = self.msqs.auto_query("test - stuff") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__content=test - stuff>") # Ensure bits in exact matches get escaped properly as well. sqs = self.msqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__content="pants:rule">') # Now with a different fieldname sqs = self.msqs.auto_query("test search -stuff", fieldname="title") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND title__content=test search -stuff>") sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname="title") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND title__content=test "my thing" search -stuff>') def test_count(self): self.assertEqual(self.msqs.count(), 23) def test_facet_counts(self): self.assertEqual(self.msqs.facet_counts(), {}) def test_best_match(self): self.assertTrue(isinstance(self.msqs.best_match(), SearchResult)) def test_latest(self): self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult)) def test_more_like_this(self): mock = MockModel() mock.id = 1 self.assertEqual(len(self.msqs.more_like_this(mock)), 23) def test_facets(self): sqs = self.msqs.facet("foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.facets), 1) sqs2 = self.msqs.facet("foo").facet("bar") self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.facets), 2) def test_date_facets(self): try: sqs = self.msqs.date_facet( "foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="smarblaph" ) self.fail() except FacetingError as e: self.assertEqual( str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second." ) sqs = self.msqs.date_facet( "foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month" ) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.date_facets), 1) sqs2 = self.msqs.date_facet( "foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month" ).date_facet("bar", start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="year") self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.date_facets), 2) def test_query_facets(self): sqs = self.msqs.query_facet("foo", "[bar TO *]") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_facets), 1) sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet("bar", "[100 TO 499]") self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.query_facets), 2) # Test multiple query facets on a single field sqs3 = ( self.msqs.query_facet("foo", "[bar TO *]") .query_facet("bar", "[100 TO 499]") .query_facet("foo", "[1000 TO 1499]") ) self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.query_facets), 3) def test_stats(self): sqs = self.msqs.stats_facet("foo", "bar") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.stats), 1) sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz") self.assertTrue(isinstance(sqs2, SearchQuerySet)) self.assertEqual(len(sqs2.query.stats), 1) sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz") self.assertTrue(isinstance(sqs3, SearchQuerySet)) self.assertEqual(len(sqs3.query.stats), 2) def test_narrow(self): sqs = self.msqs.narrow("foo:moof") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.narrow_queries), 1) def test_clone(self): results = self.msqs.filter(foo="bar", foo__lt="10") clone = results._clone() self.assertTrue(isinstance(clone, SearchQuerySet)) self.assertEqual(str(clone.query), str(results.query)) self.assertEqual(clone._result_cache, []) self.assertEqual(clone._result_count, None) self.assertEqual(clone._cache_full, False) self.assertEqual(clone._using, results._using) def test_using(self): sqs = SearchQuerySet(using="default") self.assertNotEqual(sqs.query, None) self.assertEqual(sqs.query._using, "default") def test_chaining(self): sqs = self.msqs.filter(content="foo") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) # A second instance should inherit none of the changes from above. sqs = self.msqs.filter(content="bar") self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 1) def test_none(self): sqs = self.msqs.none() self.assertTrue(isinstance(sqs, EmptySearchQuerySet)) self.assertEqual(len(sqs), 0) def test___and__(self): sqs1 = self.msqs.filter(content="foo") sqs2 = self.msqs.filter(content="bar") sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) def test___or__(self): sqs1 = self.msqs.filter(content="foo") sqs2 = self.msqs.filter(content="bar") sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) def test_and_or(self): """ Combining AND queries with OR should give AND(OR(a, b), OR(c, d)) """ sqs1 = self.msqs.filter(content="foo").filter(content="oof") sqs2 = self.msqs.filter(content="bar").filter(content="rab") sqs = sqs1 | sqs2 self.assertEqual(sqs.query.query_filter.connector, "OR") self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)) self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter)) def test_or_and(self): """ Combining OR queries with AND should give OR(AND(a, b), AND(c, d)) """ sqs1 = self.msqs.filter(content="foo").filter_or(content="oof") sqs2 = self.msqs.filter(content="bar").filter_or(content="rab") sqs = sqs1 & sqs2 self.assertEqual(sqs.query.query_filter.connector, "AND") self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)) self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter))
def _parse_facets(qd): """ Given a request, returns a quadruple of: * A SearchQuerySet constrained by the request.GET vars. * a dict of constraints that have been applied * a list of potential choices for new constraints * a querystring that produces these constraints (without pagination or sorting) """ sqs = SearchQuerySet() constraints = [] # Process constraints, and execute facets for name, field in settings.ALL_FIELDS.iteritems(): if field['document']: terms = qd.get(field['name'], qd.get('document_', None)) if terms: sqs = sqs.auto_query(terms).highlight() constraints.append({ 'value': terms, 'key': name, 'type': 'document', 'display': field['display_name'], }) elif field['faceted']: sqs = sqs.raw_params(**{ 'f.%s_exact.facet.limit' % field['name']: field['facet_limit'] }) if field['type'] == 'date': sqs, start, end = _narrow_range(sqs, field, qd) if start: constraints.append({ 'value': start.strftime("%Y-%m-%d"), 'key': name + '__gte', 'type': 'date', 'display': "%s - more than" % field['display_name'], }) if end: constraints.append({ 'value': end.strftime("%Y-%m-%d"), 'key': name + '__lte', 'type': 'date', 'display': "%s - less than" % field['display_name'], }) if start or end or name in settings.FACET_DISPLAY: if not end: end = utils.d_to_dt(getattr(SearchQuerySet().order_by("-" + name)[0], name)) if not start: start = utils.d_to_dt(getattr(SearchQuerySet().order_by(name)[0], name)) span = max(1, (end - start).days) gap = max(1, int(span / 100)) sqs = sqs.date_facet(name, start, end, 'day', gap) elif field['type'] in ('int', 'float', 'latitude', 'longitude'): sqs, start, end = _narrow_range(sqs, field, qd) if start: constraints.append({ 'value': start, 'key': name + '__gte', 'type': 'min_max', 'display': "%s - more than" % field['display_name'], }) if end: constraints.append({ 'value': end, 'key': name + '__lte', 'type': 'min_max', 'display': "%s - less than" % field['display_name'], }) if start or end or name in settings.FACET_DISPLAY: sqs = sqs.facet(name) # if not end: # end = getattr(SearchQuerySet().order_by("-" + name)[0], name) # if not start: # start = getattr(SearchQuerySet().order_by(name)[0], name) # span = max(1, (end - start)) # gap = max(1, int(span / 100)) # exact = "%s_exact" % name # sqs = sqs.raw_params(**{ # "facet.range": exact, # "f.%s.facet.range.start" % exact: start, # "f.%s.facet.range.end" % exact: end, # "f.%s.facet.range.gap" % exact: gap, # }) else: val = qd.get(name, None) if val is not None: constraints.append({ 'value': val, 'key': name, 'type': 'text', 'display': field['display_name'], }) sqs = sqs.narrow('%s:"%s"' % (name + "_exact", val)) if val is not None or name in settings.FACET_DISPLAY: sqs = sqs.facet(name) # Constraint removal links queryargs = dict((p['key'], p['value']) for p in constraints) querystr = urllib.urlencode(queryargs) for param in constraints: # remove key value = queryargs.pop(param['key']) remainder = urllib.urlencode(queryargs) param.update({'remove_link': remainder}) # restore key queryargs[param['key']] = value # Choices counts = sqs.facet_counts() choices = [] for name in settings.FACET_DISPLAY: field = settings.ALL_FIELDS[name] choice = {'field': field} if field['document']: choice.update({ 'type': 'document', 'value': queryargs.get(name, ''), }) elif field['type'] in ('text', 'char'): facets = sorted((k, c) for k, c in counts['fields'][name] if c > 0) if facets: choice.update({ 'choices': facets, 'type': 'text', 'value': queryargs.get(name, '') }) elif field['type'] in ('int', 'float', 'latitude', 'longitude'): facets = sorted([(int(k), c) for k,c in counts['fields'][name] if c > 0]) if facets: choice.update({ 'type': 'min_max', 'counts': [c for k,c in facets], 'vals': [k for k,c in facets], 'min_value': facets[0][0], 'max_value': facets[-1][0], 'chosen_min': queryargs.get(name + '__gte', ''), 'chosen_max': queryargs.get(name + '__lte', ''), }) elif field['type'] == 'date': facets = sorted(counts['dates'].get(name, {}).iteritems()) if facets: val_counts = [] vals = [] last_dt = None for d,c in facets: if c > 0: try: last_dt = utils.iso_to_datetime(d) val_counts.append(c) vals.append(last_dt.strftime('%Y-%m-%d')) except (TypeError, ValueError): pass if vals and last_dt: max_value = min( utils.iso_to_datetime(counts['dates'][name]['end']), utils.d_to_dt( getattr(SearchQuerySet().order_by('-' + name)[0], name) ) + datetime.timedelta(days=1), last_dt + datetime.timedelta( days=int(re.sub('[^\d]', '', counts['dates'][name]['gap'])) ), ) vals.append(max_value.strftime('%Y-%m-%d')) val_counts.append(0) choice.update({ 'type': 'date', 'counts': val_counts, 'vals': vals, 'min_value': vals[0], 'max_value': vals[-1], 'chosen_min': queryargs.get(name + '__gte', ''), 'chosen_max': queryargs.get(name + '__lte', ''), }) choices.append(choice) return sqs, choices, constraints, querystr