def _haystack_search(self, content, queryset, max_results, partial, using): from haystack.query import RelatedSearchQuerySet from haystack.inputs import AutoQuery if content.strip() == '': return RelatedSearchQuerySet().none() # Limit to the model bound to this manager, e.g. DataConcept. # `load_all` ensures a single database hit when loading the objects # that match sqs = RelatedSearchQuerySet().models(self.model).load_all() # If a non-default backend is being used, set which backend is to # be used. if using is not None: sqs = sqs.using(using) if partial: # Autocomplete only works with N-gram fields. sqs = sqs.autocomplete(text_auto=content) else: # Automatically handles advanced search syntax for negations # and quoted strings. sqs = sqs.filter(text=AutoQuery(content)) if queryset is not None: sqs = sqs.load_all_queryset(self.model, queryset) if max_results: return sqs[:max_results] return sqs
def search_products( request, template_name='satchless/search/haystack_predictive/products.html'): form = forms.ProductPredictiveSearchForm(data=request.GET or None) if form.is_valid(): results = form.search() query = form.cleaned_data['q'] else: results = RelatedSearchQuerySet().all() results = results.load_all() results = results.models(Product) query = '' products_per_page = getattr(settings, 'HAYSTACK_PREDICTIVE_PRODUCTS_PER_PAGE', PRODUCTS_PER_PAGE) paginator = Paginator(results, products_per_page) try: page = paginator.page(request.GET.get('page', 1)) except InvalidPage: raise Http404() return direct_to_template(request, template_name, { 'form': form, 'page': page, 'paginator': paginator, 'query': query, })
def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # With the models registered, you get the proper bits. import haystack from haystack.sites import SearchSite # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_site = haystack.site test_site = SearchSite() test_site.register(MockModel, SolrMockModelSearchIndex) haystack.site = test_site self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. mockmodel_index = test_site.get_index(MockModel) mockmodel_index.update()
def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = SolrMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. self.smmi.update()
def search(self, content, queryset=None, max_results=10): from haystack.query import RelatedSearchQuerySet sqs = RelatedSearchQuerySet().models(self.model).load_all()\ .auto_query(content) if queryset is not None: sqs = sqs.load_all_queryset(self.model, queryset) if max_results: return sqs[:max_results] return sqs
def plus_search(tags, search, search_types, order, extra_filter=None): items = get_resources_for_tag_intersection(tags) q = None for typ, info in search_types: if info[0]: typ_items = Q(**info[0]) if info[1]: typ_items = typ_items & ~Q(**info[1]) elif info[1]: typ_items = ~Q(**info[1]) if not q: q = typ_items else: q = q | typ_items if extra_filter: q = q & Q(**extra_filter) if q: items = items.filter(q) results_map = {} tag_intersection = [] if search: results = RelatedSearchQuerySet().auto_query(search) results_map = {} if results: all_results = results.load_all() all_results = all_results.load_all_queryset(GenericReference, items) results_map["All"] = [item.object for item in all_results] # we really really shouldn't do this else: results_map = {"All": EmptySearchQuerySet()} else: if items: results_map["All"] = items.all() if "All" in results_map: tag_intersection = get_intersecting_tags(results_map["All"], n=15) if len(search_types) > 1: for typ, info in search_types: if info[0]: typ_items = items.filter(**info[0]) if info[1]: typ_items = typ_items.exclude(**info[1]) elif info[1]: typ_items = items.exclude(**info[1]) if search and results: typ_items = all_results.load_all_queryset(GenericReference, typ_items) typ_items = [item.object for item in typ_items] # we really really shouldn't do this results_map[typ] = typ_items else: results_map = {"All": EmptySearchQuerySet()} search_types = [ (typ, data[2], results_map[typ], len(results_map[typ])) for typ, data in search_types if results_map.has_key(typ) ] return (results_map["All"], search_types, tag_intersection)
def create_queryset(user): # ideally we'd do it this way, and not mention specific models here # (instead doing visibility checking in their respective search_index.py files), # but there's no way to pass a user into serach_index.load_all_queryset. yet. # # return RelatedSearchQuerySet() qs = RelatedSearchQuerySet().load_all_queryset(GroupTopic, GroupTopic.objects.visible(user)) qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user)) qs = qs.load_all_queryset(Event, Event.objects.visible(user)) return qs
def get_queryset(self): query = self.request.GET.get("query", "") order_by = self.request.GET.get("order", "") try: queryset = ( RelatedSearchQuerySet().models(ForumMessage).autocomplete( auto=html.escape(query))) except TypeError: return [] if order_by == "date": queryset = queryset.order_by("-date") queryset = queryset.load_all() queryset = queryset.load_all_queryset( ForumMessage, ForumMessage.objects.all().prefetch_related( "topic__forum__edit_groups").prefetch_related( "topic__forum__view_groups").prefetch_related( "topic__forum__owner_club"), ) # Filter unauthorized responses resp = [] count = 0 max_count = 30 for r in queryset: if count >= max_count: return resp if can_view(r.object, self.request.user) and can_view( r.object.topic, self.request.user): resp.append(r.object) count += 1 return resp
def __init__(self, GET, *args, **kwargs): # Save all GET parameters in case they match facets and not form fields self.filters = GET super(FeedSearchForm, self).__init__(GET, *args, **kwargs) self.searcher = Searcher(model=Trial, facets=my_facet_config, queryset=RelatedSearchQuerySet())
def get_results(self, request): if not SEARCH_VAR in request.GET: return super(AllChangeList, self).get_results(request) # Note that pagination is 0-based, not 1-based. sqs = RelatedSearchQuerySet().models(self.model).auto_query( request.GET[SEARCH_VAR]).load_all() if not request.user.is_superuser: result = self.model.objects.distinct().filter( Q(owners=request.user) | Q(editor_groups__name__in=request.user.groups.values_list( 'name', flat=True))) else: result = self.model.objects.distinct() if 'storage_object__publication_status__exact' in request.GET: result = result.filter( storage_object__publication_status__exact=request. GET["storage_object__publication_status__exact"]) sqs = sqs.load_all_queryset(self.model, result) paginator = Paginator(sqs, self.list_per_page) # Fixed bug for pagination, count full_result_count full_result_count = 0 for sq in sqs: full_result_count += 1 result_count = paginator.count # Get the number of objects, with admin filters applied. can_show_all = result_count <= list_max_show_all(self) multi_page = result_count > self.list_per_page # Get the list of objects to display on this page. try: result_list = paginator.page(self.page_num + 1).object_list # Grab just the Django models, since that's what everything else is # expecting. result_list = [result.object for result in result_list] except InvalidPage: result_list = () self.result_count = result_count self.full_result_count = full_result_count self.result_list = result_list self.can_show_all = can_show_all self.multi_page = multi_page self.paginator = paginator
def search(self, query): if not query: return EmptySearchQuerySet() # OPTIMIZE: the number of public projects can increase substantially # causing a really high number of project_ids to be sended to # elasticsearch projects = Project.objects.public_or_collaborate(self.request.user) return RelatedSearchQuerySet()\ .filter(project_id__in=projects.values_list('pk', flat=True))\ .filter(SQ(content__contains=Clean(query)) | SQ(title__contains=Clean(query)))
def get_results(self, request): if not SEARCH_VAR in request.GET: return super(AllChangeList, self).get_results(request) # Note that pagination is 0-based, not 1-based. sqs = RelatedSearchQuerySet().models(self.model).auto_query(request.GET[SEARCH_VAR]).load_all() if not request.user.is_superuser: result = self.model.objects.distinct().filter(Q(owners=request.user) | Q(editor_groups__name__in= request.user.groups.values_list('name', flat=True))) else: result = self.model.objects.distinct() if 'storage_object__publication_status__exact' in request.GET: result = result.filter(storage_object__publication_status__exact=request.GET["storage_object__publication_status__exact"]) sqs = sqs.load_all_queryset(self.model, result) paginator = Paginator(sqs, self.list_per_page) # Fixed bug for pagination, count full_result_count full_result_count = 0 for sq in sqs: full_result_count += 1 result_count = paginator.count # Get the number of objects, with admin filters applied. can_show_all = result_count <= list_max_show_all(self) multi_page = result_count > self.list_per_page # Get the list of objects to display on this page. try: result_list = paginator.page(self.page_num+1).object_list # Grab just the Django models, since that's what everything else is # expecting. result_list = [result.object for result in result_list] except InvalidPage: result_list = () self.result_count = result_count self.full_result_count = full_result_count self.result_list = result_list self.can_show_all = can_show_all self.multi_page = multi_page self.paginator = paginator
def research_list(request, study_type="all", template="participant/research_list.html", extra_context=None): if request.method == "POST": q = request.POST.get("q", "") research_list = RelatedSearchQuerySet().filter(context=AutoQuery(q)).load_all() if study_type == "ol": research_list = research_list.load_all_queryset( Research, Research.objects.filter(is_on_web=True, is_publish=True, is_complete=False) .exclude(participantresearch__participant=request.user) .exclude(scientistresearch__scientist=request.user), ) elif study_type == "nol" or study_type == "nol_map": research_list = research_list.load_all_queryset( Research, Research.objects.filter(is_on_web=False, is_publish=True, is_complete=False) .exclude(participantresearch__participant=request.user) .exclude(scientistresearch__scientist=request.user), ) elif study_type == "all": research_list = research_list.load_all_queryset( Research, Research.objects.filter(is_publish=True, is_complete=False) .exclude(participantresearch__participant=request.user) .exclude(scientistresearch__scientist=request.user), ) paginator = get_page(request, research_list, 15) context = {"study_type": study_type, "paginator": paginator, "research_list": research_list, "q": q} else: research_list = research_randomize(request, study_type) paginator = get_page(request, research_list, 15) context = {"study_type": study_type, "paginator": paginator, "research_list": research_list, "q": ""} if extra_context: context.update(extra_context) return render_to_response(template, context, context_instance=RequestContext(request))
def search_products(request, template_name='satchless/search/haystack_predictive/products.html'): form = forms.ProductPredictiveSearchForm(data=request.GET or None) if form.is_valid(): results = form.search() query = form.cleaned_data['q'] else: results = RelatedSearchQuerySet().all() results = results.load_all() results = results.models(Product) query = '' products_per_page = getattr(settings, 'HAYSTACK_PREDICTIVE_PRODUCTS_PER_PAGE', PRODUCTS_PER_PAGE) paginator = Paginator(results, products_per_page) try: page = paginator.page(request.GET.get('page', 1)) except InvalidPage: raise Http404() return TemplateResponse(request, template_name, { 'form': form, 'page': page, 'paginator': paginator, 'query': query, })
class CustomSearchView(SearchView): def __init__(self, *args, **kwargs): kwargs["form_class"] = SpeechForm super(CustomSearchView, self).__init__(*args, **kwargs) def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get("sort") == "newest": self.searchqueryset = self.searchqueryset.order_by("-start_date") return super(CustomSearchView, self).build_form(*args, **kwargs) def extra_context(self): if not self.query: return {} self.form_class = SpeakerForm person_form = self.build_form() return {"speaker_results": person_form.search()}
class CustomSearchView(SearchView): def __init__(self, *args, **kwargs): kwargs['form_class'] = SpeechForm super(CustomSearchView, self).__init__(*args, **kwargs) def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get('sort') != 'relevance': self.searchqueryset = self.searchqueryset.order_by('-start_date') return super(CustomSearchView, self).build_form(*args, **kwargs) def extra_context(self): if not self.query: return {} self.form_class = SpeakerForm person_form = self.build_form() return { 'speaker_results': person_form.search(), }
class CustomSearchView(SearchView): def __init__(self, *args, **kwargs): kwargs['form_class'] = SpeechForm super(CustomSearchView, self).__init__(*args, **kwargs) def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get('sort') == 'newest': self.searchqueryset = self.searchqueryset.order_by('-start_date') return super(CustomSearchView, self).build_form(*args, **kwargs) def extra_context(self): if not self.query: return {} self.form_class = SpeakerForm person_form = self.build_form() return { 'speaker_results': person_form.search(), }
def setUp(self): super(LiveSolrRelatedSearchQuerySetTestCase, self).setUp() # With the models registered, you get the proper bits. import haystack from haystack.sites import SearchSite # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_site = haystack.site test_site = SearchSite() test_site.register(MockModel) haystack.site = test_site self.rsqs = RelatedSearchQuerySet() # Force indexing of the content. for mock in MockModel.objects.all(): mock.save()
def filter_queryset(self, request, queryset, view): current_user = request.user permissioned_qs = None if current_user.is_staff: permissioned_qs = queryset elif not current_user.is_anonymous(): user_profile = UserProfile.objects.get(user=current_user) permissioned_qs = queryset.filter( Q(parent_id=user_profile.id, parent_type=ContentType.objects.get_for_model(UserProfile)) | Q(parent_id__in=user_profile.organizations, parent_type=ContentType.objects.get_for_model( Organization)) | ~Q(public_access=ACCESS_TYPE_NONE)) else: permissioned_qs = queryset.filter(~Q( public_access=ACCESS_TYPE_NONE)) return super(ConceptContainerPermissionedSearchFilter, self)._filter_queryset(request, permissioned_qs, view, RelatedSearchQuerySet())
def create_queryset(user): # ideally we'd do it this way, and not mention specific models here # (instead doing visibility checking in their respective search_index.py files), # but there's no way to pass a user into serach_index.load_all_queryset. yet. # # return RelatedSearchQuerySet() qs = RelatedSearchQuerySet().load_all_queryset( GroupTopic, GroupTopic.objects.visible(user)) qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user)) qs = qs.load_all_queryset(Event, Event.objects.visible(user)) # TODO: centralize this somewhere? execlist = get_object_or_none(Community, slug='exec') if execlist and execlist.user_is_member(user, True): qs = qs.load_all_queryset(Activity, Activity.objects.all()) else: qs = qs.load_all_queryset(Activity, Activity.objects.none()) return qs
def create_queryset(user): # ideally we'd do it this way, and not mention specific models here # (instead doing visibility checking in their respective search_index.py files), # but there's no way to pass a user into serach_index.load_all_queryset. yet. # # return RelatedSearchQuerySet() qs = RelatedSearchQuerySet().load_all_queryset(GroupTopic, GroupTopic.objects.visible(user)) qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user)) qs = qs.load_all_queryset(Event, Event.objects.visible(user)) qs = qs.load_all_queryset(Term, Term.objects.all()) # TODO: centralize this somewhere? execlist = get_object_or_none(Community, slug='exec') if execlist and execlist.user_is_member(user, True): qs = qs.load_all_queryset(Activity, Activity.objects.all()) else: qs = qs.load_all_queryset(Activity, Activity.objects.none()) return qs
class LiveSolrSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the SearchQuerySet.""" fixtures = ['bulk_data.json'] def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = SolrMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. self.smmi.update() def tearDown(self): # Restore. connections['default']._index = self.old_ui settings.DEBUG = self.old_debug super(LiveSolrSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.sqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:") def test_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(connections['default'].queries), 1) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(connections['default'].queries), 1) def test_count(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() self.assertEqual(sqs.count(), 23) self.assertEqual(sqs.count(), 23) self.assertEqual(len(sqs), 23) self.assertEqual(sqs.count(), 23) # Should only execute one query to count the length of the result set. self.assertEqual(len(connections['default'].queries), 1) def test_manual_iter(self): results = self.sqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 3) def test_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.sqs._cache_is_full(), False) results = self.sqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 3) def test___and__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo AND bar)') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar') sqs = sqs3 & sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 3) self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)') def test___or__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo OR bar)') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar').models(MockModel) sqs = sqs3 | sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)') def test_auto_query(self): # Ensure bits in exact matches get escaped properly as well. # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants\\:rule>') self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"') self.assertEqual(len(sqs), 0) # Regressions def test_regression_proper_start_offsets(self): sqs = self.sqs.filter(text='index') self.assertNotEqual(sqs.count(), 0) id_counts = {} for item in sqs: if item.id in id_counts: id_counts[item.id] += 1 else: id_counts[item.id] = 1 for key, value in id_counts.items(): if value > 1: self.fail("Result with id '%s' seen more than once in the results." % key) def test_regression_raw_search_breaks_slicing(self): sqs = self.sqs.raw_search('text: index') page_1 = [result.pk for result in sqs[0:10]] page_2 = [result.pk for result in sqs[10:20]] for pk in page_2: if pk in page_1: self.fail("Result with id '%s' seen more than once in the results." % pk) # RelatedSearchQuerySet Tests def test_related_load_all(self): sqs = self.rsqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:") def test_related_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(2, 24)) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(11, 24)) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 4) def test_related_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(connections['default'].queries), 3) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(connections['default'].queries), 4) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23]) self.assertEqual(len(connections['default'].queries), 4) def test_related_manual_iter(self): results = self.rsqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 4) def test_related_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_related_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 5) def test_quotes_regression(self): sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E") # Should not have empty terms. self.assertEqual(sqs.query.build_query(), u"(44\ufffd\ufffd48'40''N AND 20\ufffd\ufffd28'32''E)") # Should not cause Solr to 500. self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing') self.assertEqual(sqs.query.build_query(), u'blazing') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing saddles') self.assertEqual(sqs.query.build_query(), u'(blazing AND saddles)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles') self.assertEqual(sqs.query.build_query(), u'(\\"blazing AND saddles)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles"') self.assertEqual(sqs.query.build_query(), u'"blazing saddles"') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles"') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'saddles"') self.assertEqual(sqs.query.build_query(), u'("blazing \'saddles" AND mel)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"') self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'') self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel AND \')') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"') self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel AND \'\\")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel brooks') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND brooks)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" brooks') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND brooks)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND \\"brooks)') self.assertEqual(sqs.count(), 0) def test_result_class(self): # Assert that we're defaulting to ``SearchResult``. sqs = self.sqs.all() self.assertTrue(isinstance(sqs[0], SearchResult)) # Custom class. sqs = self.sqs.result_class(MockSearchResult).all() self.assertTrue(isinstance(sqs[0], MockSearchResult)) # Reset to default. sqs = self.sqs.result_class(None).all() self.assertTrue(isinstance(sqs[0], SearchResult))
class LiveSolrSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the SearchQuerySet.""" fixtures = ['bulk_data.json'] def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # With the models registered, you get the proper bits. import haystack from haystack.sites import SearchSite # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_site = haystack.site test_site = SearchSite() test_site.register(MockModel, SolrMockModelSearchIndex) haystack.site = test_site self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. mockmodel_index = test_site.get_index(MockModel) mockmodel_index.update() def tearDown(self): # Restore. import haystack haystack.site = self.old_site settings.DEBUG = self.old_debug super(LiveSolrSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.sqs.load_all() self.assert_(isinstance(sqs, SearchQuerySet)) self.assert_(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:") def test_iter(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) sqs = self.sqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 3) def test_slice(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.sqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(backends.queries), 1) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.sqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(backends.queries), 1) def test_manual_iter(self): results = self.sqs.all() backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 3) def test_fill_cache(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.sqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(backends.queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(backends.queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(backends.queries), 2) def test_cache_is_full(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) self.assertEqual(self.sqs._cache_is_full(), False) results = self.sqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(backends.queries), 3) def test___and__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 & sqs2 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo AND bar)') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar') sqs = sqs3 & sqs4 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 3) self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)') def test___or__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 | sqs2 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo OR bar)') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar').models(MockModel) sqs = sqs3 | sqs4 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)') def test_auto_query(self): # Ensure bits in exact matches get escaped properly as well. # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants\\:rule>') self.assertEqual(sqs.query.build_query(), u'pants\\:rule') self.assertEqual(len(sqs), 0) # Regressions def test_regression_proper_start_offsets(self): sqs = self.sqs.filter(text='index') self.assertNotEqual(sqs.count(), 0) id_counts = {} for item in sqs: if item.id in id_counts: id_counts[item.id] += 1 else: id_counts[item.id] = 1 for key, value in id_counts.items(): if value > 1: self.fail("Result with id '%s' seen more than once in the results." % key) def test_regression_raw_search_breaks_slicing(self): sqs = self.sqs.raw_search('text: index') page_1 = [result.pk for result in sqs[0:10]] page_2 = [result.pk for result in sqs[10:20]] for pk in page_2: if pk in page_1: self.fail("Result with id '%s' seen more than once in the results." % pk) # RelatedSearchQuerySet Tests def test_related_load_all(self): sqs = self.rsqs.load_all() self.assert_(isinstance(sqs, SearchQuerySet)) self.assert_(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:") def test_related_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(2, 24)) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(11, 24)) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) def test_related_iter(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 4) def test_related_slice(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(backends.queries), 3) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(backends.queries), 4) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23]) self.assertEqual(len(backends.queries), 4) def test_related_manual_iter(self): results = self.rsqs.all() backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 4) def test_related_fill_cache(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(backends.queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(backends.queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(backends.queries), 2) def test_related_cache_is_full(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(backends.queries), 5)
class LiveSolrSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the SearchQuerySet.""" fixtures = ['bulk_data.json'] def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = SolrMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. self.smmi.update() def tearDown(self): # Restore. connections['default']._index = self.old_ui settings.DEBUG = self.old_debug super(LiveSolrSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.sqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual( sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:" ) def test_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(connections['default'].queries), 1) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(connections['default'].queries), 1) def test_count(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() self.assertEqual(sqs.count(), 23) self.assertEqual(sqs.count(), 23) self.assertEqual(len(sqs), 23) self.assertEqual(sqs.count(), 23) # Should only execute one query to count the length of the result set. self.assertEqual(len(connections['default'].queries), 1) def test_manual_iter(self): results = self.sqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 3) def test_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.sqs._cache_is_full(), False) results = self.sqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 3) def test___and__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))') # Now for something more complex... sqs3 = self.sqs.exclude( title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar') sqs = sqs3 & sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 3) self.assertEqual( sqs.query.build_query(), u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))') def test___or__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))') # Now for something more complex... sqs3 = self.sqs.exclude( title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar').models(MockModel) sqs = sqs3 | sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual( sqs.query.build_query(), u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))') def test_auto_query(self): # Ensure bits in exact matches get escaped properly as well. # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">') self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")') self.assertEqual(len(sqs), 0) # Regressions def test_regression_proper_start_offsets(self): sqs = self.sqs.filter(text='index') self.assertNotEqual(sqs.count(), 0) id_counts = {} for item in sqs: if item.id in id_counts: id_counts[item.id] += 1 else: id_counts[item.id] = 1 for key, value in id_counts.items(): if value > 1: self.fail( "Result with id '%s' seen more than once in the results." % key) def test_regression_raw_search_breaks_slicing(self): sqs = self.sqs.raw_search('text: index') page_1 = [result.pk for result in sqs[0:10]] page_2 = [result.pk for result in sqs[10:20]] for pk in page_2: if pk in page_1: self.fail( "Result with id '%s' seen more than once in the results." % pk) # RelatedSearchQuerySet Tests def test_related_load_all(self): sqs = self.rsqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual( sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:" ) def test_related_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(2, 24)) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(11, 24)) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 4) def test_related_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(connections['default'].queries), 3) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(connections['default'].queries), 4) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23]) self.assertEqual(len(connections['default'].queries), 4) def test_related_manual_iter(self): results = self.rsqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 4) def test_related_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_related_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 5) def test_quotes_regression(self): sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E") # Should not have empty terms. self.assertEqual(sqs.query.build_query(), u"(44\xb048'40''N 20\xb028'32''E)") # Should not cause Solr to 500. self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing') self.assertEqual(sqs.query.build_query(), u'(blazing)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing saddles') self.assertEqual(sqs.query.build_query(), u'(blazing saddles)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles') self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles"') self.assertEqual(sqs.query.build_query(), u'("blazing saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'saddles"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \')') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \'\\")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel brooks') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel brooks)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" brooks') self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" brooks)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" \\"brooks)') self.assertEqual(sqs.count(), 0) def test_query_generation(self): sqs = self.sqs.filter( SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world"))) self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))") def test_result_class(self): # Assert that we're defaulting to ``SearchResult``. sqs = self.sqs.all() self.assertTrue(isinstance(sqs[0], SearchResult)) # Custom class. sqs = self.sqs.result_class(MockSearchResult).all() self.assertTrue(isinstance(sqs[0], MockSearchResult)) # Reset to default. sqs = self.sqs.result_class(None).all() self.assertTrue(isinstance(sqs[0], SearchResult))
def plus_search(tags, search, search_types, order=None, in_group=None, extra_filter=None): items = get_resources_for_tag_intersection(tags) q = None for typ, info in search_types: if info[0]: typ_items = Q(**info[0]) if info[1]: typ_items = typ_items & ~Q(**info[1]) elif info[1]: typ_items = ~Q(**info[1]) if not q: q = typ_items else: q = q | typ_items if extra_filter: q = q & Q(**extra_filter) if q: items = items.filter(q) included = None for obj_class, included_filter in object_type_filters.items(): objs = obj_class.objects.filter(**included_filter) included_search = {'content_type__model':obj_class.__name__.lower(), 'object_id__in':objs} if not included: included = Q(**included_search) included = included | Q(**included_search) items = items.filter(included) if in_group: # this should be implemented using the code just above and an external search filter arg page_ids = WikiPage.objects.filter(in_agent=in_group, stub=False) wikipages = Q(**{'content_type__model':'wikipage', 'object_id__in':page_ids}) resource_ids = Resource.objects.filter(in_agent=in_group, stub=False) resources = Q(**{'content_type__model':'resource', 'object_id__in':resource_ids}) q = resources | wikipages items = items.filter(q) results_map = {} tag_intersection = [] if search: results = RelatedSearchQuerySet().auto_query(search) results_map = {} if results: all_results = results.load_all_queryset(GenericReference, items) if order == 'relevance': all_results = all_results.load_all() # this bit is quite evil and makes things really inefficient for large searches # a better approach would be to get all the ids directly from the fulltext index and use them as a filter for GenericReferences results_map['All'] = [item.object for item in all_results] #we really really shouldn't do this items_len = len(results_map['All']) else: search_results = [result.pk for result in all_results] results_map['All'] = items.filter(id__in=search_results).order_by(order) items_len = results_map['All'].count() else: results_map = {'All':EmptySearchQuerySet()} items_len = 0 else: if items: items = items.order_by('creator') items_len = items.count() results_map['All'] = items else: items_len = 0 results_map = {'All':EmptySearchQuerySet()} if order == 'modified': results_map['All'] = results_map['All'].order_by('-' + order) elif order == 'display_name': results_map['All'] = results_map['All'].order_by(order) if 'All' in results_map: tag_intersection = get_intersecting_tags(results_map['All'], n=15) if len(search_types) > 1: for typ, info in search_types: if info[0]: typ_items = items.filter(**info[0]) if info[1]: typ_items = items.exclude(**info[1]) if search and results and order == 'relevance': # why do this again when we could just separate results using python typ_items = all_results.load_all_queryset(GenericReference, typ_items) typ_items = [item.object for item in typ_items] #we really really shouldn't do this if typ_items: results_map[typ] = typ_items else: results_map = {'All':EmptySearchQuerySet()} search_type_data = [] for typ, data in search_types: if results_map.has_key(typ): try: type_len = results_map[typ].count() except TypeError: type_len = len(results_map[typ]) search_type_data.append((typ, data[2], results_map[typ], type_len)) return {'All':results_map['All'], 'items_len':items_len, 'search_types':search_type_data, 'tag_intersection':tag_intersection}
def filter_queryset(self, request, queryset, view): return self._filter_queryset(request, queryset, view, RelatedSearchQuerySet())
def search(request): """ Returns messages corresponding to a query """ mlist_fqdn = request.GET.get("mlist") if mlist_fqdn is None: mlist = None else: try: mlist = MailingList.objects.get(name=mlist_fqdn) except MailingList.DoesNotExist: raise Http404("No archived mailing-list by that name.") if not is_mlist_authorized(request, mlist): return render(request, "hyperkitty/errors/private.html", { "mlist": mlist, }, status=403) query = '' results = EmptySearchQuerySet() sqs = RelatedSearchQuerySet() # Remove private non-subscribed lists if mlist is not None: sqs = sqs.filter(mailinglist__exact=mlist.name) else: excluded_mlists = MailingList.objects.filter( archive_policy=ArchivePolicy.private.value) if request.user.is_authenticated(): subscriptions = request.user.hyperkitty_profile.get_subscriptions() excluded_mlists = excluded_mlists.exclude( name__in=subscriptions.keys()) excluded_mlists = excluded_mlists.values_list("name", flat=True) sqs = sqs.exclude(mailinglist__in=excluded_mlists) # Sorting sort_mode = request.GET.get('sort') if sort_mode == "date-asc": sqs = sqs.order_by("date") elif sort_mode == "date-desc": sqs = sqs.order_by("-date") # Handle data if request.GET.get('q'): form = SearchForm( request.GET, searchqueryset=sqs, load_all=True) if form.is_valid(): query = form.cleaned_data['q'] results = form.search() else: form = SearchForm(searchqueryset=sqs, load_all=True) messages = paginate(results, page_num=request.GET.get('page')) for message in messages: if request.user.is_authenticated(): message.object.myvote = message.object.votes.filter(user=request.user).first() else: message.object.myvote = None context = { 'mlist' : mlist, 'form': form, 'messages': messages, 'query': query, 'sort_mode': sort_mode, 'suggestion': None, } if results.query.backend.include_spelling: context['suggestion'] = form.get_suggestion() return render(request, "hyperkitty/search_results.html", context)
def plus_search(tags, search, search_types, order=None, in_group=None, extra_filter=None): items = get_resources_for_tag_intersection(tags) q = None for typ, info in search_types: if info[0]: typ_items = Q(**info[0]) if info[1]: typ_items = typ_items & ~Q(**info[1]) elif info[1]: typ_items = ~Q(**info[1]) if not q: q = typ_items else: q = q | typ_items if extra_filter: q = q & Q(**extra_filter) if q: items = items.filter(q) included = None for obj_class, included_filter in object_type_filters.items(): objs = obj_class.objects.filter(**included_filter) included_search = { 'content_type__model': obj_class.__name__.lower(), 'object_id__in': objs } if not included: included = Q(**included_search) included = included | Q(**included_search) items = items.filter(included) if in_group: # this should be implemented using the code just above and an external search filter arg page_ids = WikiPage.objects.filter(in_agent=in_group, stub=False) wikipages = Q(**{ 'content_type__model': 'wikipage', 'object_id__in': page_ids }) resource_ids = Resource.objects.filter(in_agent=in_group, stub=False) resources = Q(**{ 'content_type__model': 'resource', 'object_id__in': resource_ids }) q = resources | wikipages items = items.filter(q) results_map = {} tag_intersection = [] if search: results = RelatedSearchQuerySet().auto_query(search) results_map = {} if results: all_results = results.load_all_queryset(GenericReference, items) if order == 'relevance': all_results = all_results.load_all( ) # this bit is quite evil and makes things really inefficient for large searches # a better approach would be to get all the ids directly from the fulltext index and use them as a filter for GenericReferences results_map['All'] = [item.object for item in all_results ] #we really really shouldn't do this items_len = len(results_map['All']) else: search_results = [result.pk for result in all_results] results_map['All'] = items.filter( id__in=search_results).order_by(order) items_len = results_map['All'].count() else: results_map = {'All': EmptySearchQuerySet()} items_len = 0 else: if items: items = items.order_by('creator') items_len = items.count() results_map['All'] = items else: items_len = 0 results_map = {'All': EmptySearchQuerySet()} if order == 'modified': results_map['All'] = results_map['All'].order_by('-' + order) elif order == 'display_name': results_map['All'] = results_map['All'].order_by(order) if 'All' in results_map: tag_intersection = get_intersecting_tags(results_map['All'], n=15) if len(search_types) > 1: for typ, info in search_types: if info[0]: typ_items = items.filter(**info[0]) if info[1]: typ_items = items.exclude(**info[1]) if search and results and order == 'relevance': # why do this again when we could just separate results using python typ_items = all_results.load_all_queryset( GenericReference, typ_items) typ_items = [item.object for item in typ_items ] #we really really shouldn't do this if typ_items: results_map[typ] = typ_items else: results_map = {'All': EmptySearchQuerySet()} search_type_data = [] for typ, data in search_types: if results_map.has_key(typ): try: type_len = results_map[typ].count() except TypeError: type_len = len(results_map[typ]) search_type_data.append((typ, data[2], results_map[typ], type_len)) return { 'All': results_map['All'], 'items_len': items_len, 'search_types': search_type_data, 'tag_intersection': tag_intersection }
def test_empty(self): self.assertEqual(len(DataConcept.objects.published()), 12) self.assertEqual(RelatedSearchQuerySet().models(DataConcept).count(), 12) self.assertEqual(len(DataConcept.objects.search('')), 0)
def get_context_data(self, **kwargs): context = super(SAQuestionIndex, self).get_context_data(**kwargs) context['ministers'] = [] ministers = Section.objects.filter(parent__heading='Questions') ministers = sorted(ministers, key=questions_section_sort_key) for minister in ministers: context['ministers'].append({ 'title': minister.title.replace('Questions asked to the ', ''), 'slug': minister.slug }) context['orderby'] = 'recentanswers' context['minister'] = 'all' context['q'] = '' #set filter values for key in ('orderby', 'minister', 'q'): if key in self.request.GET: context[key] = self.request.GET[key] if not context['orderby'] in ['recentquestions', 'recentanswers']: context['orderby'] = 'recentquestions' search_result_sections = [] if context['q'] != '': #using a RelatedSearchQuerySet seems to result in fewer #queries, although the same results can be achieved with a #SearchQuerySet query = RelatedSearchQuerySet().models(Speech) query = query.filter( tags__name__in=['question', 'answer'], content=AutoQuery(context['q']), ).load_all() all_speeches = Speech.objects.all().filter( tags__name__in=['question', 'answer']) if context['minister'] != 'all': all_speeches = all_speeches.filter( section__parent__slug=context['minister']) query = query.load_all_queryset(Speech, all_speeches) for result in query: search_result_sections.append(result.object.section.id) sections = Section \ .objects \ .filter( parent__parent__heading='Questions' ) \ .select_related('parent') \ .prefetch_related('speech_set') \ .annotate( earliest_date=Min('speech__start_date'), smallest_id=Min('speech__id'), number_speeches=Count('speech'), latest_date=Max('speech__start_date'), ) if context['minister'] != 'all': sections = sections.filter(parent__slug=context['minister']) if len(search_result_sections) > 0: sections = sections.filter(id__in=search_result_sections) if context['orderby'] == 'recentanswers': sections = sections.filter(number_speeches__gt=1).order_by( '-latest_date', '-smallest_id') else: sections = sections.order_by('-earliest_date', '-smallest_id') paginator = Paginator(sections, 10) page = self.request.GET.get('page') try: sections = paginator.page(page) except PageNotAnInteger: sections = paginator.page(1) except EmptyPage: sections = paginator.page(paginator.num_pages) context['paginator'] = sections #format questions and answers for the template questions = [] for section in sections: question = section.speech_set.all()[0] question.questionto = section.parent.heading.replace( 'Questions asked to the ', '') question.questionto_slug = section.parent.slug #assume if there is more than one speech that the question is answered if len(section.speech_set.all()) > 1: question.answer = section \ .speech_set \ .all()[len(section.speech_set.all()) - 1] #extract the actual reply from the reply text (replies #often include the original question and other text, #such as a letterhead) text = question.answer.text.split('REPLY') if len(text) > 1: question.answer.text = text[len(text) - 1] if question.answer.text[0] == ':': question.answer.text = question.answer.text[1:] questions.append(question) context['speeches'] = questions return context
"events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND " " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'" " GROUP BY events_event.id", 'audio_count': "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE " "events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND " " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='audio'" " GROUP BY events_event.id", })) return sqs event_search = EventSearchView(form_class=EventSearchForm, searchqueryset=RelatedSearchQuerySet()) def annotate_events(events): return events.annotate(product_count=Count('products')).extra( select={ 'video_count': "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE " "events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND " " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'" " GROUP BY events_event.id", 'audio_count': "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE " "events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND "
class LiveSolrRelatedSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the RelatedSearchQuerySet.""" fixtures = ["solr_bulk_data.json"] def setUp(self): super(LiveSolrRelatedSearchQuerySetTestCase, self).setUp() # With the models registered, you get the proper bits. import haystack from haystack.sites import SearchSite # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_site = haystack.site test_site = SearchSite() test_site.register(MockModel) haystack.site = test_site self.rsqs = RelatedSearchQuerySet() # Force indexing of the content. for mock in MockModel.objects.all(): mock.save() def tearDown(self): # Restore. import haystack haystack.site = self.old_site settings.DEBUG = self.old_debug super(LiveSolrRelatedSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.rsqs.load_all() self.assert_(isinstance(sqs, SearchQuerySet)) self.assert_(len(sqs) > 0) self.assertEqual( sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:", ) def test_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(2, 24)) def test_iter(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 4) def test_slice(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(backends.queries), 3) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(backends.queries), 4) def test_manual_iter(self): results = self.rsqs.all() backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 4) def test_fill_cache(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(backends.queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(backends.queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(backends.queries), 2) def test_cache_is_full(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(backends.queries), 5)
def search(request): """ Returns messages corresponding to a query """ mlist_fqdn = request.GET.get("mlist") if mlist_fqdn is None: mlist = None else: try: mlist = MailingList.objects.get(name=mlist_fqdn) except MailingList.DoesNotExist: raise Http404("No archived mailing-list by that name.") if not is_mlist_authorized(request, mlist): return render(request, "hyperkitty/errors/private.html", { "mlist": mlist, }, status=403) query = '' results = EmptySearchQuerySet() sqs = RelatedSearchQuerySet() # Remove private non-subscribed lists if mlist is not None: sqs = sqs.filter(mailinglist__exact=mlist.name) else: excluded_mlists = MailingList.objects.filter( archive_policy=ArchivePolicy.private.value) if request.user.is_authenticated: subscriptions = get_subscriptions(request.user) excluded_mlists = excluded_mlists.exclude( list_id__in=subscriptions.keys()) excluded_mlists = excluded_mlists.values_list("name", flat=True) sqs = sqs.exclude(mailinglist__in=excluded_mlists) # Sorting sort_mode = request.GET.get('sort') if sort_mode == "date-asc": sqs = sqs.order_by("date") elif sort_mode == "date-desc": sqs = sqs.order_by("-date") # Handle data if request.GET.get('q'): form = SearchForm(request.GET, searchqueryset=sqs, load_all=True) if form.is_valid(): query = form.cleaned_data['q'] results = form.search() else: form = SearchForm(searchqueryset=sqs, load_all=True) try: emails = paginate( results, request.GET.get('page'), request.GET.get('count'), ) except Exception as e: backend = settings.HAYSTACK_CONNECTIONS[DEFAULT_ALIAS]["ENGINE"] if backend == "haystack.backends.whoosh_backend.WhooshEngine": from whoosh.qparser.common import QueryParserError search_exception = QueryParserError if backend == "xapian_backend.XapianEngine": from xapian import QueryParserError search_exception = QueryParserError if not isinstance(e, search_exception): raise emails = paginate([]) form.add_error( "q", ValidationError( _('Parsing error: %(error)s'), params={"error": e}, code="parse", )) for email in emails: if request.user.is_authenticated: email.object.myvote = email.object.votes.filter( user=request.user).first() else: email.object.myvote = None context = { 'mlist': mlist, 'form': form, 'emails': emails, 'query': query, 'sort_mode': sort_mode, 'suggestion': None, } if results.query.backend.include_spelling: context['suggestion'] = form.get_suggestion() return render(request, "hyperkitty/search_results.html", context)
def __init__(self, *args, **kwargs): if kwargs.get('searchqueryset') is None: kwargs['searchqueryset'] = RelatedSearchQuerySet() super(SpeechForm, self).__init__(*args, **kwargs)
def get_query(self, search_term): return RelatedSearchQuerySet().filter(content=search_term)
def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get('sort') == 'newest': self.searchqueryset = self.searchqueryset.order_by('-start_date') return super(CustomSearchView, self).build_form(*args, **kwargs)
def __init__(self, *args, **kwargs): if not kwargs.get('searchqueryset', None): kwargs['searchqueryset'] = RelatedSearchQuerySet() super(ProductPredictiveSearchForm, self).__init__(*args, **kwargs)
def get_context_data(self, **kwargs): context = super(SAQuestionIndex, self).get_context_data(**kwargs) context['ministers'] = [] ministers = Section.objects.filter(parent__heading='Questions') ministers = sorted(ministers, key=questions_section_sort_key) for minister in ministers: context['ministers'].append({ 'title': minister.title.replace('Questions asked to the ', ''), 'slug': minister.slug }) context['orderby'] = 'recentanswers' context['minister'] = 'all' context['q'] = '' #set filter values for key in ('orderby', 'minister', 'q'): if key in self.request.GET: context[key] = self.request.GET[key] if not context['orderby'] in ['recentquestions', 'recentanswers']: context['orderby'] = 'recentquestions' search_result_sections = [] if context['q'] != '': #using a RelatedSearchQuerySet seems to result in fewer #queries, although the same results can be achieved with a #SearchQuerySet query = RelatedSearchQuerySet().models(Speech) query = query.filter( tags__name__in=['question', 'answer'], content=AutoQuery(context['q']), ).load_all() all_speeches = Speech.objects.all().filter( tags__name__in=['question', 'answer']) if context['minister'] != 'all': all_speeches = all_speeches.filter( section__parent__slug=context['minister'] ) query = query.load_all_queryset(Speech, all_speeches) for result in query: search_result_sections.append(result.object.section.id) sections = Section \ .objects \ .filter( parent__parent__heading='Questions' ) \ .select_related('parent__heading') \ .prefetch_related('speech_set') \ .annotate( earliest_date=Min('speech__start_date'), smallest_id=Min('speech__id'), number_speeches=Count('speech'), latest_date=Max('speech__start_date'), ) if context['minister'] != 'all': sections = sections.filter(parent__slug=context['minister']) if len(search_result_sections) > 0: sections = sections.filter(id__in=search_result_sections) if context['orderby'] == 'recentanswers': sections = sections.filter(number_speeches__gt=1).order_by( '-latest_date', '-smallest_id' ) else: sections = sections.order_by( '-earliest_date', '-smallest_id' ) paginator = Paginator(sections, 10) page = self.request.GET.get('page') try: sections = paginator.page(page) except PageNotAnInteger: sections = paginator.page(1) except EmptyPage: sections = paginator.page(paginator.num_pages) context['paginator'] = sections #format questions and answers for the template questions = [] for section in sections: question = section.speech_set.all()[0] question.questionto = section.parent.heading.replace( 'Questions asked to the ', '') question.questionto_slug = section.parent.slug #assume if there is more than one speech that the question is answered if len(section.speech_set.all()) > 1: question.answer = section \ .speech_set \ .all()[len(section.speech_set.all()) - 1] #extract the actual reply from the reply text (replies #often include the original question and other text, #such as a letterhead) text = question.answer.text.split('REPLY') if len(text) > 1: question.answer.text = text[len(text) - 1] if question.answer.text[0] == ':': question.answer.text = question.answer.text[1:] questions.append(question) context['speeches'] = questions return context
class LiveElasticsearchSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the SearchQuerySet.""" fixtures = ['bulk_data.json'] def setUp(self): super(LiveElasticsearchSearchQuerySetTestCase, self).setUp() # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = ElasticsearchMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_elasticsearch_index() # Force indexing of the content. self.smmi.update() def tearDown(self): # Restore. connections['default']._index = self.old_ui settings.DEBUG = self.old_debug super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.sqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.') def test_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() results = sorted([int(result.pk) for result in sqs]) self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10]) self.assertEqual(len(connections['default'].queries), 1) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(int(results[21].pk), 18) self.assertEqual(len(connections['default'].queries), 1) def test_count(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() self.assertEqual(sqs.count(), 23) self.assertEqual(sqs.count(), 23) self.assertEqual(len(sqs), 23) self.assertEqual(sqs.count(), 23) # Should only execute one query to count the length of the result set. self.assertEqual(len(connections['default'].queries), 1) def test_manual_iter(self): results = self.sqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20]) self.assertEqual(len(connections['default'].queries), 3) def test_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.sqs._cache_is_full(), False) results = self.sqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 3) def test___and__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo AND bar)') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar') sqs = sqs3 & sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 3) self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)') def test___or__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo OR bar)') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar').models(MockModel) sqs = sqs3 | sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)') def test_auto_query(self): # Ensure bits in exact matches get escaped properly as well. # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">') self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"') self.assertEqual(len(sqs), 0) # Regressions def test_regression_proper_start_offsets(self): sqs = self.sqs.filter(text='index') self.assertNotEqual(sqs.count(), 0) id_counts = {} for item in sqs: if item.id in id_counts: id_counts[item.id] += 1 else: id_counts[item.id] = 1 for key, value in id_counts.items(): if value > 1: self.fail("Result with id '%s' seen more than once in the results." % key) def test_regression_raw_search_breaks_slicing(self): sqs = self.sqs.raw_search('text:index') page_1 = [result.pk for result in sqs[0:10]] page_2 = [result.pk for result in sqs[10:20]] for pk in page_2: if pk in page_1: self.fail("Result with id '%s' seen more than once in the results." % pk) # RelatedSearchQuerySet Tests def test_related_load_all(self): sqs = self.rsqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.') def test_related_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual(sorted([obj.object.id for obj in sqs]), range(2, 24)) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], [12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20]) self.assertEqual([obj.object.id for obj in sqs[10:20]], [13, 18, 20]) def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20]) self.assertEqual(len(connections['default'].queries), 4) def test_related_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10]) self.assertEqual(len(connections['default'].queries), 3) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 18) self.assertEqual(len(connections['default'].queries), 4) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[20:30]], [13, 18, 20]) self.assertEqual(len(connections['default'].queries), 4) def test_related_manual_iter(self): results = self.rsqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = sorted([int(result.pk) for result in results._manual_iter()]) self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 4) def test_related_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_related_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 5) def test_quotes_regression(self): sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E") # Should not have empty terms. self.assertEqual(sqs.query.build_query(), u"44\xb048'40''N 20\xb028'32''E") # Should not cause Elasticsearch to 500. self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing') self.assertEqual(sqs.query.build_query(), u'blazing') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing saddles') self.assertEqual(sqs.query.build_query(), u'blazing saddles') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles') self.assertEqual(sqs.query.build_query(), u'\\"blazing saddles') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles"') self.assertEqual(sqs.query.build_query(), u'"blazing saddles"') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles"') self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles"') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'saddles"') self.assertEqual(sqs.query.build_query(), u'mel "blazing \'saddles"') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"') self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles"') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'') self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles" \'') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"') self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles" \'\\"') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel') self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel brooks') self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel brooks') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" brooks') self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles" brooks') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles" \\"brooks') self.assertEqual(sqs.count(), 0) def test_result_class(self): # Assert that we're defaulting to ``SearchResult``. sqs = self.sqs.all() self.assertTrue(isinstance(sqs[0], SearchResult)) # Custom class. sqs = self.sqs.result_class(MockSearchResult).all() self.assertTrue(isinstance(sqs[0], MockSearchResult)) # Reset to default. sqs = self.sqs.result_class(None).all() self.assertTrue(isinstance(sqs[0], SearchResult))
def search(request): """ Returns messages corresponding to a query """ mlist_fqdn = request.GET.get("mlist") if mlist_fqdn is None: mlist = None else: try: mlist = MailingList.objects.get(name=mlist_fqdn) except MailingList.DoesNotExist: raise Http404("No archived mailing-list by that name.") if not is_mlist_authorized(request, mlist): return render(request, "hyperkitty/errors/private.html", { "mlist": mlist, }, status=403) query = '' results = EmptySearchQuerySet() sqs = RelatedSearchQuerySet() # Remove private non-subscribed lists if mlist is not None: sqs = sqs.filter(mailinglist__exact=mlist.name) else: excluded_mlists = MailingList.objects.filter( archive_policy=ArchivePolicy.private.value) if request.user.is_authenticated(): subscriptions = request.user.hyperkitty_profile.get_subscriptions() excluded_mlists = excluded_mlists.exclude( list_id__in=subscriptions.keys()) excluded_mlists = excluded_mlists.values_list("name", flat=True) sqs = sqs.exclude(mailinglist__in=excluded_mlists) # Sorting sort_mode = request.GET.get('sort') if sort_mode == "date-asc": sqs = sqs.order_by("date") elif sort_mode == "date-desc": sqs = sqs.order_by("-date") # Handle data if request.GET.get('q'): form = SearchForm( request.GET, searchqueryset=sqs, load_all=True) if form.is_valid(): query = form.cleaned_data['q'] results = form.search() else: form = SearchForm(searchqueryset=sqs, load_all=True) emails = paginate(results, page_num=request.GET.get('page')) for email in emails: if request.user.is_authenticated(): email.object.myvote = email.object.votes.filter(user=request.user).first() else: email.object.myvote = None context = { 'mlist' : mlist, 'form': form, 'emails': emails, 'query': query, 'sort_mode': sort_mode, 'suggestion': None, } if results.query.backend.include_spelling: context['suggestion'] = form.get_suggestion() return render(request, "hyperkitty/search_results.html", context)
class LiveElasticsearchSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the SearchQuerySet.""" fixtures = ['bulk_data.json'] def setUp(self): super(LiveElasticsearchSearchQuerySetTestCase, self).setUp() # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = ElasticsearchMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: lssqstc_all_loaded = True # Wipe it clean. clear_elasticsearch_index() # Force indexing of the content. self.smmi.update() def tearDown(self): # Restore. connections['default']._index = self.old_ui settings.DEBUG = self.old_debug super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.sqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.') def test_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() results = sorted([int(result.pk) for result in sqs]) self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 3) def test_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10]) self.assertEqual(len(connections['default'].queries), 1) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(int(results[21].pk), 18) self.assertEqual(len(connections['default'].queries), 1) def test_count(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.sqs.all() self.assertEqual(sqs.count(), 23) self.assertEqual(sqs.count(), 23) self.assertEqual(len(sqs), 23) self.assertEqual(sqs.count(), 23) # Should only execute one query to count the length of the result set. self.assertEqual(len(connections['default'].queries), 1) def test_manual_iter(self): results = self.sqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20]) self.assertEqual(len(connections['default'].queries), 3) def test_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.sqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.sqs._cache_is_full(), False) results = self.sqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 3) def test___and__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 & sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar') sqs = sqs3 & sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 3) self.assertEqual(sqs.query.build_query(), u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))') def test___or__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 | sqs2 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))') # Now for something more complex... sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar').models(MockModel) sqs = sqs3 | sqs4 self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))') def test_auto_query(self): # Ensure bits in exact matches get escaped properly as well. # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">') self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")') self.assertEqual(len(sqs), 0) # Regressions @unittest.expectedFailure def test_regression_proper_start_offsets(self): sqs = self.sqs.filter(text='index') self.assertNotEqual(sqs.count(), 0) id_counts = {} for item in sqs: if item.id in id_counts: id_counts[item.id] += 1 else: id_counts[item.id] = 1 for key, value in id_counts.items(): if value > 1: self.fail("Result with id '%s' seen more than once in the results." % key) def test_regression_raw_search_breaks_slicing(self): sqs = self.sqs.raw_search('text:index') page_1 = [result.pk for result in sqs[0:10]] page_2 = [result.pk for result in sqs[10:20]] for pk in page_2: if pk in page_1: self.fail("Result with id '%s' seen more than once in the results." % pk) # RelatedSearchQuerySet Tests def test_related_load_all(self): sqs = self.rsqs.load_all() self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertTrue(len(sqs) > 0) self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.') def test_related_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual(sorted([obj.object.id for obj in sqs]), range(2, 24)) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) self.assertTrue(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], [12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20]) self.assertEqual([obj.object.id for obj in sqs[10:20]], [13, 18, 20]) def test_related_iter(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20]) self.assertEqual(len(connections['default'].queries), 4) def test_related_slice(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10]) self.assertEqual(len(connections['default'].queries), 3) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 18) self.assertEqual(len(connections['default'].queries), 4) reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[20:30]], [13, 18, 20]) self.assertEqual(len(connections['default'].queries), 4) def test_related_manual_iter(self): results = self.rsqs.all() reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = sorted([int(result.pk) for result in results._manual_iter()]) self.assertEqual(results, range(1, 24)) self.assertEqual(len(connections['default'].queries), 4) def test_related_fill_cache(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(connections['default'].queries), 0) results._fill_cache(0, 10) self.assertEqual(len([result for result in results._result_cache if result is not None]), 10) self.assertEqual(len(connections['default'].queries), 1) results._fill_cache(10, 20) self.assertEqual(len([result for result in results._result_cache if result is not None]), 20) self.assertEqual(len(connections['default'].queries), 2) def test_related_cache_is_full(self): reset_search_queries() self.assertEqual(len(connections['default'].queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(connections['default'].queries), 5) def test_quotes_regression(self): sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E") # Should not have empty terms. self.assertEqual(sqs.query.build_query(), u"(44\xb048'40''N 20\xb028'32''E)") # Should not cause Elasticsearch to 500. self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing') self.assertEqual(sqs.query.build_query(), u'(blazing)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('blazing saddles') self.assertEqual(sqs.query.build_query(), u'(blazing saddles)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles') self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles"') self.assertEqual(sqs.query.build_query(), u'("blazing saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'saddles"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \')') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"') self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \'\\")') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('"blazing saddles" mel brooks') self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel brooks)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" brooks') self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" brooks)') self.assertEqual(sqs.count(), 0) sqs = self.sqs.auto_query('mel "blazing saddles" "brooks') self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" \\"brooks)') self.assertEqual(sqs.count(), 0) def test_query_generation(self): sqs = self.sqs.filter(SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world"))) self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))") def test_result_class(self): # Assert that we're defaulting to ``SearchResult``. sqs = self.sqs.all() self.assertTrue(isinstance(sqs[0], SearchResult)) # Custom class. sqs = self.sqs.result_class(MockSearchResult).all() self.assertTrue(isinstance(sqs[0], MockSearchResult)) # Reset to default. sqs = self.sqs.result_class(None).all() self.assertTrue(isinstance(sqs[0], SearchResult))
def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get("sort") == "newest": self.searchqueryset = self.searchqueryset.order_by("-start_date") return super(CustomSearchView, self).build_form(*args, **kwargs)
handler500 = "pinax.views.server_error" urlpatterns = patterns("", url(r"^", include("pins.urls")), url(r"^admin/", include(admin.site.urls)), url(r'^admin_tools/', include('admin_tools.urls')), url(r"^about/", include("about.urls")), url(r"^account/", include("account.urls")), url(r"^profile/", include("profiles.urls")), url(r'^comments/', include('django.contrib.comments.urls')), url('^activity/', include('actstream.urls')), url(r'^abuse/', include('abuse_reports.urls')), url(r'^invite/', include('invite_friends.urls')), url(r'^auth/', include('social_auth.urls')), url(r'^cms/', include('cms.urls')), ) urlpatterns = urlpatterns + patterns('haystack.views', url(r'^search/$', SearchView(searchqueryset=RelatedSearchQuerySet()), name='haystack_search'), ) if settings.SERVE_MEDIA: urlpatterns += patterns("", url(r"", include("staticfiles.urls"),), url(r'^site_media/media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}), url(r'^site_media/static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT}), )
class LiveSolrSearchQuerySetTestCase(TestCase): """Used to test actual implementation details of the SearchQuerySet.""" fixtures = ['bulk_data.json'] def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # With the models registered, you get the proper bits. import haystack from haystack.sites import SearchSite # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_site = haystack.site test_site = SearchSite() test_site.register(MockModel, SolrMockModelSearchIndex) haystack.site = test_site self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. mockmodel_index = test_site.get_index(MockModel) mockmodel_index.update() def tearDown(self): # Restore. import haystack haystack.site = self.old_site settings.DEBUG = self.old_debug super(LiveSolrSearchQuerySetTestCase, self).tearDown() def test_load_all(self): sqs = self.sqs.load_all() self.assert_(isinstance(sqs, SearchQuerySet)) self.assert_(len(sqs) > 0) self.assertEqual( sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:" ) def test_iter(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) sqs = self.sqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 3) def test_slice(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.sqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(backends.queries), 1) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.sqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(backends.queries), 1) def test_count(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) sqs = self.sqs.all() self.assertEqual(sqs.count(), 23) self.assertEqual(sqs.count(), 23) self.assertEqual(len(sqs), 23) self.assertEqual(sqs.count(), 23) # Should only execute one query to count the length of the result set. self.assertEqual(len(backends.queries), 1) def test_manual_iter(self): results = self.sqs.all() backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 3) def test_fill_cache(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.sqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(backends.queries), 0) results._fill_cache(0, 10) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 10) self.assertEqual(len(backends.queries), 1) results._fill_cache(10, 20) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 20) self.assertEqual(len(backends.queries), 2) def test_cache_is_full(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) self.assertEqual(self.sqs._cache_is_full(), False) results = self.sqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(backends.queries), 3) def test___and__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 & sqs2 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo AND bar)') # Now for something more complex... sqs3 = self.sqs.exclude( title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar') sqs = sqs3 & sqs4 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 3) self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)') def test___or__(self): sqs1 = self.sqs.filter(content='foo') sqs2 = self.sqs.filter(content='bar') sqs = sqs1 | sqs2 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'(foo OR bar)') # Now for something more complex... sqs3 = self.sqs.exclude( title='moof').filter(SQ(content='foo') | SQ(content='baz')) sqs4 = self.sqs.filter(content='bar').models(MockModel) sqs = sqs3 | sqs4 self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs.query.query_filter), 2) self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)') def test_auto_query(self): # Ensure bits in exact matches get escaped properly as well. # This will break horrifically if escaping isn't working. sqs = self.sqs.auto_query('"pants:rule"') self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants\\:rule>') self.assertEqual(sqs.query.build_query(), u'pants\\:rule') self.assertEqual(len(sqs), 0) # Regressions def test_regression_proper_start_offsets(self): sqs = self.sqs.filter(text='index') self.assertNotEqual(sqs.count(), 0) id_counts = {} for item in sqs: if item.id in id_counts: id_counts[item.id] += 1 else: id_counts[item.id] = 1 for key, value in id_counts.items(): if value > 1: self.fail( "Result with id '%s' seen more than once in the results." % key) def test_regression_raw_search_breaks_slicing(self): sqs = self.sqs.raw_search('text: index') page_1 = [result.pk for result in sqs[0:10]] page_2 = [result.pk for result in sqs[10:20]] for pk in page_2: if pk in page_1: self.fail( "Result with id '%s' seen more than once in the results." % pk) # RelatedSearchQuerySet Tests def test_related_load_all(self): sqs = self.rsqs.load_all() self.assert_(isinstance(sqs, SearchQuerySet)) self.assert_(len(sqs) > 0) self.assertEqual( sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:" ) def test_related_load_all_queryset(self): sqs = self.rsqs.load_all() self.assertEqual(len(sqs._load_all_querysets), 0) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1)) self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(2, 24)) sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10)) self.assert_(isinstance(sqs, SearchQuerySet)) self.assertEqual(len(sqs._load_all_querysets), 1) self.assertEqual([obj.object.id for obj in sqs], range(11, 24)) self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23]) def test_related_iter(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) sqs = self.rsqs.all() results = [int(result.pk) for result in sqs] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 4) def test_related_slice(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]) self.assertEqual(len(backends.queries), 3) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual(int(results[21].pk), 22) self.assertEqual(len(backends.queries), 4) backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23]) self.assertEqual(len(backends.queries), 4) def test_related_manual_iter(self): results = self.rsqs.all() backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = [int(result.pk) for result in results._manual_iter()] self.assertEqual(results, range(1, 24)) self.assertEqual(len(backends.queries), 4) def test_related_fill_cache(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) results = self.rsqs.all() self.assertEqual(len(results._result_cache), 0) self.assertEqual(len(backends.queries), 0) results._fill_cache(0, 10) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 10) self.assertEqual(len(backends.queries), 1) results._fill_cache(10, 20) self.assertEqual( len([ result for result in results._result_cache if result is not None ]), 20) self.assertEqual(len(backends.queries), 2) def test_related_cache_is_full(self): backends.reset_search_queries() self.assertEqual(len(backends.queries), 0) self.assertEqual(self.rsqs._cache_is_full(), False) results = self.rsqs.all() fire_the_iterator_and_fill_cache = [result for result in results] self.assertEqual(results._cache_is_full(), True) self.assertEqual(len(backends.queries), 5)
def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get('sort') != 'relevance': self.searchqueryset = self.searchqueryset.order_by('-start_date') return super(CustomSearchView, self).build_form(*args, **kwargs)