Exemplo n.º 1
0
    def _haystack_search(self, content, queryset, max_results, partial, using):
        from haystack.query import RelatedSearchQuerySet
        from haystack.inputs import AutoQuery

        if content.strip() == '':
            return RelatedSearchQuerySet().none()

        # Limit to the model bound to this manager, e.g. DataConcept.
        # `load_all` ensures a single database hit when loading the objects
        # that match
        sqs = RelatedSearchQuerySet().models(self.model).load_all()

        # If a non-default backend is being used, set which backend is to
        # be used.
        if using is not None:
            sqs = sqs.using(using)

        if partial:
            # Autocomplete only works with N-gram fields.
            sqs = sqs.autocomplete(text_auto=content)
        else:
            # Automatically handles advanced search syntax for negations
            # and quoted strings.
            sqs = sqs.filter(text=AutoQuery(content))

        if queryset is not None:
            sqs = sqs.load_all_queryset(self.model, queryset)

        if max_results:
            return sqs[:max_results]

        return sqs
Exemplo n.º 2
0
def search_products(
        request,
        template_name='satchless/search/haystack_predictive/products.html'):
    form = forms.ProductPredictiveSearchForm(data=request.GET or None)
    if form.is_valid():
        results = form.search()
        query = form.cleaned_data['q']
    else:
        results = RelatedSearchQuerySet().all()
        results = results.load_all()
        results = results.models(Product)
        query = ''

    products_per_page = getattr(settings,
                                'HAYSTACK_PREDICTIVE_PRODUCTS_PER_PAGE',
                                PRODUCTS_PER_PAGE)
    paginator = Paginator(results, products_per_page)
    try:
        page = paginator.page(request.GET.get('page', 1))
    except InvalidPage:
        raise Http404()

    return direct_to_template(request, template_name, {
        'form': form,
        'page': page,
        'paginator': paginator,
        'query': query,
    })
Exemplo n.º 3
0
    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()

        # With the models registered, you get the proper bits.
        import haystack
        from haystack.sites import SearchSite

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_site = haystack.site
        test_site = SearchSite()
        test_site.register(MockModel, SolrMockModelSearchIndex)
        haystack.site = test_site

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_solr_index()

            # Force indexing of the content.
            mockmodel_index = test_site.get_index(MockModel)
            mockmodel_index.update()
Exemplo n.º 4
0
    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = SolrMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_solr_index()

            # Force indexing of the content.
            self.smmi.update()
Exemplo n.º 5
0
 def search(self, content, queryset=None, max_results=10):
     from haystack.query import RelatedSearchQuerySet
     sqs = RelatedSearchQuerySet().models(self.model).load_all()\
         .auto_query(content)
     if queryset is not None:
         sqs = sqs.load_all_queryset(self.model, queryset)
     if max_results:
         return sqs[:max_results]
     return sqs
Exemplo n.º 6
0
def plus_search(tags, search, search_types, order, extra_filter=None):
    items = get_resources_for_tag_intersection(tags)
    q = None
    for typ, info in search_types:
        if info[0]:
            typ_items = Q(**info[0])
            if info[1]:
                typ_items = typ_items & ~Q(**info[1])
        elif info[1]:
            typ_items = ~Q(**info[1])
        if not q:
            q = typ_items
        else:
            q = q | typ_items
    if extra_filter:
        q = q & Q(**extra_filter)
    if q:
        items = items.filter(q)

    results_map = {}
    tag_intersection = []
    if search:
        results = RelatedSearchQuerySet().auto_query(search)
        results_map = {}
        if results:
            all_results = results.load_all()
            all_results = all_results.load_all_queryset(GenericReference, items)
            results_map["All"] = [item.object for item in all_results]  # we really really shouldn't do this
        else:
            results_map = {"All": EmptySearchQuerySet()}
    else:
        if items:
            results_map["All"] = items.all()

    if "All" in results_map:
        tag_intersection = get_intersecting_tags(results_map["All"], n=15)
        if len(search_types) > 1:
            for typ, info in search_types:
                if info[0]:
                    typ_items = items.filter(**info[0])
                    if info[1]:
                        typ_items = typ_items.exclude(**info[1])
                elif info[1]:
                    typ_items = items.exclude(**info[1])
                if search and results:
                    typ_items = all_results.load_all_queryset(GenericReference, typ_items)
                    typ_items = [item.object for item in typ_items]  # we really really shouldn't do this
                    results_map[typ] = typ_items
    else:
        results_map = {"All": EmptySearchQuerySet()}

    search_types = [
        (typ, data[2], results_map[typ], len(results_map[typ]))
        for typ, data in search_types
        if results_map.has_key(typ)
    ]
    return (results_map["All"], search_types, tag_intersection)
Exemplo n.º 7
0
def create_queryset(user):
    # ideally we'd do it this way, and not mention specific models here
    # (instead doing visibility checking in their respective search_index.py files),
    # but there's no way to pass a user into serach_index.load_all_queryset.  yet.
    #   
    # return RelatedSearchQuerySet()

    qs = RelatedSearchQuerySet().load_all_queryset(GroupTopic, GroupTopic.objects.visible(user))
    qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user))
    qs = qs.load_all_queryset(Event, Event.objects.visible(user))
    return qs
Exemplo n.º 8
0
 def setUp(self):
     super(LiveSolrSearchQuerySetTestCase, self).setUp()
     
     # With the models registered, you get the proper bits.
     import haystack
     from haystack.sites import SearchSite
     
     # Stow.
     self.old_debug = settings.DEBUG
     settings.DEBUG = True
     self.old_site = haystack.site
     test_site = SearchSite()
     test_site.register(MockModel, SolrMockModelSearchIndex)
     haystack.site = test_site
     
     self.sqs = SearchQuerySet()
     self.rsqs = RelatedSearchQuerySet()
     
     # Ugly but not constantly reindexing saves us almost 50% runtime.
     global lssqstc_all_loaded
     
     if lssqstc_all_loaded is None:
         print 'Reloading data...'
         lssqstc_all_loaded = True
         
         # Wipe it clean.
         clear_solr_index()
         
         # Force indexing of the content.
         mockmodel_index = test_site.get_index(MockModel)
         mockmodel_index.update()
Exemplo n.º 9
0
    def get_queryset(self):
        query = self.request.GET.get("query", "")
        order_by = self.request.GET.get("order", "")

        try:
            queryset = (
                RelatedSearchQuerySet().models(ForumMessage).autocomplete(
                    auto=html.escape(query)))
        except TypeError:
            return []

        if order_by == "date":
            queryset = queryset.order_by("-date")

        queryset = queryset.load_all()
        queryset = queryset.load_all_queryset(
            ForumMessage,
            ForumMessage.objects.all().prefetch_related(
                "topic__forum__edit_groups").prefetch_related(
                    "topic__forum__view_groups").prefetch_related(
                        "topic__forum__owner_club"),
        )

        # Filter unauthorized responses
        resp = []
        count = 0
        max_count = 30
        for r in queryset:
            if count >= max_count:
                return resp
            if can_view(r.object, self.request.user) and can_view(
                    r.object.topic, self.request.user):
                resp.append(r.object)
                count += 1
        return resp
    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = SolrMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_solr_index()

            # Force indexing of the content.
            self.smmi.update()
Exemplo n.º 11
0
    def __init__(self, GET, *args, **kwargs):
        # Save all GET parameters in case they match facets and not form fields
        self.filters = GET

        super(FeedSearchForm, self).__init__(GET, *args, **kwargs)
        self.searcher = Searcher(model=Trial,
                                 facets=my_facet_config,
                                 queryset=RelatedSearchQuerySet())
Exemplo n.º 12
0
    def get_results(self, request):
        if not SEARCH_VAR in request.GET:
            return super(AllChangeList, self).get_results(request)

        # Note that pagination is 0-based, not 1-based.
        sqs = RelatedSearchQuerySet().models(self.model).auto_query(
            request.GET[SEARCH_VAR]).load_all()
        if not request.user.is_superuser:
            result = self.model.objects.distinct().filter(
                Q(owners=request.user)
                | Q(editor_groups__name__in=request.user.groups.values_list(
                    'name', flat=True)))
        else:
            result = self.model.objects.distinct()
        if 'storage_object__publication_status__exact' in request.GET:
            result = result.filter(
                storage_object__publication_status__exact=request.
                GET["storage_object__publication_status__exact"])
        sqs = sqs.load_all_queryset(self.model, result)
        paginator = Paginator(sqs, self.list_per_page)
        # Fixed bug for pagination, count full_result_count
        full_result_count = 0
        for sq in sqs:
            full_result_count += 1
        result_count = paginator.count
        # Get the number of objects, with admin filters applied.

        can_show_all = result_count <= list_max_show_all(self)
        multi_page = result_count > self.list_per_page

        # Get the list of objects to display on this page.
        try:
            result_list = paginator.page(self.page_num + 1).object_list
            # Grab just the Django models, since that's what everything else is
            # expecting.
            result_list = [result.object for result in result_list]
        except InvalidPage:
            result_list = ()

        self.result_count = result_count
        self.full_result_count = full_result_count
        self.result_list = result_list
        self.can_show_all = can_show_all
        self.multi_page = multi_page
        self.paginator = paginator
Exemplo n.º 13
0
    def _haystack_search(self, content, queryset, max_results, partial, using):
        from haystack.query import RelatedSearchQuerySet
        from haystack.inputs import AutoQuery

        if content.strip() == '':
            return RelatedSearchQuerySet().none()

        # Limit to the model bound to this manager, e.g. DataConcept.
        # `load_all` ensures a single database hit when loading the objects
        # that match
        sqs = RelatedSearchQuerySet().models(self.model).load_all()

        # If a non-default backend is being used, set which backend is to
        # be used.
        if using is not None:
            sqs = sqs.using(using)

        if partial:
            # Autocomplete only works with N-gram fields.
            sqs = sqs.autocomplete(text_auto=content)
        else:
            # Automatically handles advanced search syntax for negations
            # and quoted strings.
            sqs = sqs.filter(text=AutoQuery(content))

        if queryset is not None:
            sqs = sqs.load_all_queryset(self.model, queryset)

        if max_results:
            return sqs[:max_results]

        return sqs
Exemplo n.º 14
0
 def search(self, query):
     if not query:
         return EmptySearchQuerySet()
     # OPTIMIZE: the number of public projects can increase substantially
     # causing a really high number of project_ids to be sended to
     # elasticsearch
     projects = Project.objects.public_or_collaborate(self.request.user)
     return RelatedSearchQuerySet()\
         .filter(project_id__in=projects.values_list('pk', flat=True))\
         .filter(SQ(content__contains=Clean(query)) |
                 SQ(title__contains=Clean(query)))
Exemplo n.º 15
0
 def get_results(self, request):
     if not SEARCH_VAR in request.GET:
         return super(AllChangeList, self).get_results(request)
     
     # Note that pagination is 0-based, not 1-based.
     sqs = RelatedSearchQuerySet().models(self.model).auto_query(request.GET[SEARCH_VAR]).load_all()
     if not request.user.is_superuser:
         result = self.model.objects.distinct().filter(Q(owners=request.user)
                 | Q(editor_groups__name__in=
                        request.user.groups.values_list('name', flat=True)))
     else:
         result = self.model.objects.distinct()
     if 'storage_object__publication_status__exact' in request.GET:
         result = result.filter(storage_object__publication_status__exact=request.GET["storage_object__publication_status__exact"])
     sqs = sqs.load_all_queryset(self.model, result)
     paginator = Paginator(sqs, self.list_per_page)
     # Fixed bug for pagination, count full_result_count
     full_result_count = 0
     for sq in sqs:
         full_result_count += 1
     result_count = paginator.count
     # Get the number of objects, with admin filters applied.
     
     can_show_all = result_count <= list_max_show_all(self)
     multi_page = result_count > self.list_per_page
     
     # Get the list of objects to display on this page.
     try:
         result_list = paginator.page(self.page_num+1).object_list
         # Grab just the Django models, since that's what everything else is
         # expecting.
         result_list = [result.object for result in result_list]
     except InvalidPage:
         result_list = ()
     
     self.result_count = result_count
     self.full_result_count = full_result_count
     self.result_list = result_list
     self.can_show_all = can_show_all
     self.multi_page = multi_page
     self.paginator = paginator
def research_list(request, study_type="all", template="participant/research_list.html", extra_context=None):
    if request.method == "POST":
        q = request.POST.get("q", "")

        research_list = RelatedSearchQuerySet().filter(context=AutoQuery(q)).load_all()

        if study_type == "ol":
            research_list = research_list.load_all_queryset(
                Research,
                Research.objects.filter(is_on_web=True, is_publish=True, is_complete=False)
                .exclude(participantresearch__participant=request.user)
                .exclude(scientistresearch__scientist=request.user),
            )
        elif study_type == "nol" or study_type == "nol_map":
            research_list = research_list.load_all_queryset(
                Research,
                Research.objects.filter(is_on_web=False, is_publish=True, is_complete=False)
                .exclude(participantresearch__participant=request.user)
                .exclude(scientistresearch__scientist=request.user),
            )
        elif study_type == "all":
            research_list = research_list.load_all_queryset(
                Research,
                Research.objects.filter(is_publish=True, is_complete=False)
                .exclude(participantresearch__participant=request.user)
                .exclude(scientistresearch__scientist=request.user),
            )

        paginator = get_page(request, research_list, 15)

        context = {"study_type": study_type, "paginator": paginator, "research_list": research_list, "q": q}
    else:
        research_list = research_randomize(request, study_type)
        paginator = get_page(request, research_list, 15)

        context = {"study_type": study_type, "paginator": paginator, "research_list": research_list, "q": ""}

    if extra_context:
        context.update(extra_context)
    return render_to_response(template, context, context_instance=RequestContext(request))
Exemplo n.º 17
0
def search_products(request, template_name='satchless/search/haystack_predictive/products.html'):
    form = forms.ProductPredictiveSearchForm(data=request.GET or None)
    if form.is_valid():
        results = form.search()
        query = form.cleaned_data['q']
    else:
        results = RelatedSearchQuerySet().all()
        results = results.load_all()
        results = results.models(Product)
        query = ''
    products_per_page = getattr(settings, 'HAYSTACK_PREDICTIVE_PRODUCTS_PER_PAGE', PRODUCTS_PER_PAGE)
    paginator = Paginator(results, products_per_page)
    try:
        page = paginator.page(request.GET.get('page', 1))
    except InvalidPage:
        raise Http404()
    return TemplateResponse(request, template_name, {
        'form': form,
        'page': page,
        'paginator': paginator,
        'query': query,
    })
Exemplo n.º 18
0
class CustomSearchView(SearchView):
    def __init__(self, *args, **kwargs):
        kwargs["form_class"] = SpeechForm
        super(CustomSearchView, self).__init__(*args, **kwargs)

    def build_form(self, *args, **kwargs):
        self.searchqueryset = RelatedSearchQuerySet()
        if self.request.GET.get("sort") == "newest":
            self.searchqueryset = self.searchqueryset.order_by("-start_date")
        return super(CustomSearchView, self).build_form(*args, **kwargs)

    def extra_context(self):
        if not self.query:
            return {}

        self.form_class = SpeakerForm
        person_form = self.build_form()
        return {"speaker_results": person_form.search()}
Exemplo n.º 19
0
class CustomSearchView(SearchView):
    def __init__(self, *args, **kwargs):
        kwargs['form_class'] = SpeechForm
        super(CustomSearchView, self).__init__(*args, **kwargs)

    def build_form(self, *args, **kwargs):
        self.searchqueryset = RelatedSearchQuerySet()
        if self.request.GET.get('sort') != 'relevance':
            self.searchqueryset = self.searchqueryset.order_by('-start_date')
        return super(CustomSearchView, self).build_form(*args, **kwargs)

    def extra_context(self):
        if not self.query:
            return {}

        self.form_class = SpeakerForm
        person_form = self.build_form()
        return {
            'speaker_results': person_form.search(),
        }
Exemplo n.º 20
0
class CustomSearchView(SearchView):
    def __init__(self, *args, **kwargs):
        kwargs['form_class'] = SpeechForm
        super(CustomSearchView, self).__init__(*args, **kwargs)

    def build_form(self, *args, **kwargs):
        self.searchqueryset = RelatedSearchQuerySet()
        if self.request.GET.get('sort') == 'newest':
            self.searchqueryset = self.searchqueryset.order_by('-start_date')
        return super(CustomSearchView, self).build_form(*args, **kwargs)

    def extra_context(self):
        if not self.query:
            return {}

        self.form_class = SpeakerForm
        person_form = self.build_form()
        return {
            'speaker_results': person_form.search(),
        }
Exemplo n.º 21
0
    def setUp(self):
        super(LiveSolrRelatedSearchQuerySetTestCase, self).setUp()

        # With the models registered, you get the proper bits.
        import haystack
        from haystack.sites import SearchSite

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_site = haystack.site
        test_site = SearchSite()
        test_site.register(MockModel)
        haystack.site = test_site

        self.rsqs = RelatedSearchQuerySet()

        # Force indexing of the content.
        for mock in MockModel.objects.all():
            mock.save()
Exemplo n.º 22
0
    def filter_queryset(self, request, queryset, view):
        current_user = request.user
        permissioned_qs = None

        if current_user.is_staff:
            permissioned_qs = queryset
        elif not current_user.is_anonymous():
            user_profile = UserProfile.objects.get(user=current_user)
            permissioned_qs = queryset.filter(
                Q(parent_id=user_profile.id,
                  parent_type=ContentType.objects.get_for_model(UserProfile))
                | Q(parent_id__in=user_profile.organizations,
                    parent_type=ContentType.objects.get_for_model(
                        Organization)) | ~Q(public_access=ACCESS_TYPE_NONE))
        else:
            permissioned_qs = queryset.filter(~Q(
                public_access=ACCESS_TYPE_NONE))

        return super(ConceptContainerPermissionedSearchFilter,
                     self)._filter_queryset(request, permissioned_qs, view,
                                            RelatedSearchQuerySet())
Exemplo n.º 23
0
def create_queryset(user):
    # ideally we'd do it this way, and not mention specific models here
    # (instead doing visibility checking in their respective search_index.py files),
    # but there's no way to pass a user into serach_index.load_all_queryset.  yet.
    #
    # return RelatedSearchQuerySet()

    qs = RelatedSearchQuerySet().load_all_queryset(
        GroupTopic, GroupTopic.objects.visible(user))
    qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user))
    qs = qs.load_all_queryset(Event, Event.objects.visible(user))

    # TODO: centralize this somewhere?
    execlist = get_object_or_none(Community, slug='exec')
    if execlist and execlist.user_is_member(user, True):
        qs = qs.load_all_queryset(Activity, Activity.objects.all())
    else:
        qs = qs.load_all_queryset(Activity, Activity.objects.none())

    return qs
Exemplo n.º 24
0
def create_queryset(user):
    # ideally we'd do it this way, and not mention specific models here
    # (instead doing visibility checking in their respective search_index.py files),
    # but there's no way to pass a user into serach_index.load_all_queryset.  yet.
    #   
    # return RelatedSearchQuerySet()

    qs = RelatedSearchQuerySet().load_all_queryset(GroupTopic, GroupTopic.objects.visible(user))
    qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user))
    qs = qs.load_all_queryset(Event, Event.objects.visible(user))
    qs = qs.load_all_queryset(Term, Term.objects.all())

    # TODO: centralize this somewhere?
    execlist = get_object_or_none(Community, slug='exec')
    if execlist and execlist.user_is_member(user, True):
        qs = qs.load_all_queryset(Activity, Activity.objects.all())
    else:
        qs = qs.load_all_queryset(Activity, Activity.objects.none())
        
    return qs
class LiveSolrSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['bulk_data.json']

    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = SolrMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_solr_index()

            # Force indexing of the content.
            self.smmi.update()

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        settings.DEBUG = self.old_debug
        super(LiveSolrSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.sqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:")

    def test_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(connections['default'].queries), 1)

    def test_count(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(len(sqs), 23)
        self.assertEqual(sqs.count(), 23)
        # Should only execute one query to count the length of the result set.
        self.assertEqual(len(connections['default'].queries), 1)

    def test_manual_iter(self):
        results = self.sqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 3)

    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)')

    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)')

    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.
        sqs = self.sqs.auto_query('"pants:rule"')
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants\\:rule>')
        self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"')
        self.assertEqual(len(sqs), 0)

    # Regressions

    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)

        id_counts = {}

        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1

        for key, value in id_counts.items():
            if value > 1:
                self.fail("Result with id '%s' seen more than once in the results." % key)

    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text: index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]

        for pk in page_2:
            if pk in page_1:
                self.fail("Result with id '%s' seen more than once in the results." % pk)

    # RelatedSearchQuerySet Tests

    def test_related_load_all(self):
        sqs = self.rsqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:")

    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(2, 24))

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
        self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])

    def test_related_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(connections['default'].queries), 3)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(connections['default'].queries), 4)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23])
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_manual_iter(self):
        results = self.rsqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_related_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 5)

    def test_quotes_regression(self):
        sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E")
        # Should not have empty terms.
        self.assertEqual(sqs.query.build_query(), u"(44\ufffd\ufffd48'40''N AND 20\ufffd\ufffd28'32''E)")
        # Should not cause Solr to 500.
        self.assertEqual(sqs.count(), 0)

        sqs = self.sqs.auto_query('blazing')
        self.assertEqual(sqs.query.build_query(), u'blazing')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(blazing AND saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(\\"blazing AND saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'"blazing saddles"')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'saddles"')
        self.assertEqual(sqs.query.build_query(), u'("blazing \'saddles" AND mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
        self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
        self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel AND \')')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
        self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel AND \'\\")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND \\"brooks)')
        self.assertEqual(sqs.count(), 0)

    def test_result_class(self):
        # Assert that we're defaulting to ``SearchResult``.
        sqs = self.sqs.all()
        self.assertTrue(isinstance(sqs[0], SearchResult))

        # Custom class.
        sqs = self.sqs.result_class(MockSearchResult).all()
        self.assertTrue(isinstance(sqs[0], MockSearchResult))

        # Reset to default.
        sqs = self.sqs.result_class(None).all()
        self.assertTrue(isinstance(sqs[0], SearchResult))
Exemplo n.º 26
0
class LiveSolrSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['bulk_data.json']
    
    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()
        
        # With the models registered, you get the proper bits.
        import haystack
        from haystack.sites import SearchSite
        
        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_site = haystack.site
        test_site = SearchSite()
        test_site.register(MockModel, SolrMockModelSearchIndex)
        haystack.site = test_site
        
        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()
        
        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded
        
        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True
            
            # Wipe it clean.
            clear_solr_index()
            
            # Force indexing of the content.
            mockmodel_index = test_site.get_index(MockModel)
            mockmodel_index.update()
    
    def tearDown(self):
        # Restore.
        import haystack
        haystack.site = self.old_site
        settings.DEBUG = self.old_debug
        super(LiveSolrSearchQuerySetTestCase, self).tearDown()
    
    def test_load_all(self):
        sqs = self.sqs.load_all()
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assert_(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:")
    
    def test_iter(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        sqs = self.sqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 3)
    
    def test_slice(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.sqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(backends.queries), 1)
        
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.sqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(backends.queries), 1)
    
    def test_manual_iter(self):
        results = self.sqs.all()
        
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 3)
    
    def test_fill_cache(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(backends.queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(backends.queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(backends.queries), 2)
    
    def test_cache_is_full(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(backends.queries), 3)
    
    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2
        
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')
        
        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4
        
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)')
    
    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2
        
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')
        
        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4
        
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)')
    
    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.
        sqs = self.sqs.auto_query('"pants:rule"')
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants\\:rule>')
        self.assertEqual(sqs.query.build_query(), u'pants\\:rule')
        self.assertEqual(len(sqs), 0)
    
    # Regressions
    
    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)
        
        id_counts = {}
        
        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1
        
        for key, value in id_counts.items():
            if value > 1:
                self.fail("Result with id '%s' seen more than once in the results." % key)
    
    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text: index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]
        
        for pk in page_2:
            if pk in page_1:
                self.fail("Result with id '%s' seen more than once in the results." % pk)
    
    # RelatedSearchQuerySet Tests
    
    def test_related_load_all(self):
        sqs = self.rsqs.load_all()
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assert_(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:")
    
    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)
        
        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(2, 24))
        
        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
        self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])
    
    def test_related_iter(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 4)
    
    def test_related_slice(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(backends.queries), 3)
        
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(backends.queries), 4)
        
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23])
        self.assertEqual(len(backends.queries), 4)
    
    def test_related_manual_iter(self):
        results = self.rsqs.all()
        
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 4)
    
    def test_related_fill_cache(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(backends.queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(backends.queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(backends.queries), 2)
    
    def test_related_cache_is_full(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(backends.queries), 5)
Exemplo n.º 27
0
class LiveSolrSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['bulk_data.json']

    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = SolrMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_solr_index()

            # Force indexing of the content.
            self.smmi.update()

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        settings.DEBUG = self.old_debug
        super(LiveSolrSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.sqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(
            sqs[0].object.foo,
            u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
        )

    def test_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]],
                         [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(connections['default'].queries), 1)

    def test_count(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(len(sqs), 23)
        self.assertEqual(sqs.count(), 23)
        # Should only execute one query to count the length of the result set.
        self.assertEqual(len(connections['default'].queries), 1)

    def test_manual_iter(self):
        results = self.sqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 3)

    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(
            title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(
            sqs.query.build_query(),
            u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))')

    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(
            title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(
            sqs.query.build_query(),
            u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))')

    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.
        sqs = self.sqs.auto_query('"pants:rule"')
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter),
                         '<SQ: AND content__contains="pants:rule">')
        self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")')
        self.assertEqual(len(sqs), 0)

    # Regressions

    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)

        id_counts = {}

        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1

        for key, value in id_counts.items():
            if value > 1:
                self.fail(
                    "Result with id '%s' seen more than once in the results." %
                    key)

    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text: index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]

        for pk in page_2:
            if pk in page_1:
                self.fail(
                    "Result with id '%s' seen more than once in the results." %
                    pk)

    # RelatedSearchQuerySet Tests

    def test_related_load_all(self):
        sqs = self.rsqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(
            sqs[0].object.foo,
            u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
        )

    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel,
                                    MockModel.objects.filter(id__gt=1))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(2, 24))

        sqs = sqs.load_all_queryset(MockModel,
                                    MockModel.objects.filter(id__gt=10))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
        self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])

    def test_related_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]],
                         [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(connections['default'].queries), 3)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(connections['default'].queries), 4)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[20:30]],
                         [21, 22, 23])
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_manual_iter(self):
        results = self.rsqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_related_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 5)

    def test_quotes_regression(self):
        sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
        # Should not have empty terms.
        self.assertEqual(sqs.query.build_query(),
                         u"(44\xb048'40''N 20\xb028'32''E)")
        # Should not cause Solr to 500.
        self.assertEqual(sqs.count(), 0)

        sqs = self.sqs.auto_query('blazing')
        self.assertEqual(sqs.query.build_query(), u'(blazing)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(blazing saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
        self.assertEqual(sqs.query.build_query(),
                         u'(mel "blazing \'\'saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
        self.assertEqual(sqs.query.build_query(),
                         u'(mel "blazing \'\'saddles" \')')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
        self.assertEqual(sqs.query.build_query(),
                         u'(mel "blazing \'\'saddles" \'\\")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
        self.assertEqual(sqs.query.build_query(),
                         u'("blazing saddles" mel brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
        self.assertEqual(sqs.query.build_query(),
                         u'(mel "blazing saddles" brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
        self.assertEqual(sqs.query.build_query(),
                         u'(mel "blazing saddles" \\"brooks)')
        self.assertEqual(sqs.count(), 0)

    def test_query_generation(self):
        sqs = self.sqs.filter(
            SQ(content=AutoQuery("hello world"))
            | SQ(title=AutoQuery("hello world")))
        self.assertEqual(sqs.query.build_query(),
                         u"((hello world) OR title:(hello world))")

    def test_result_class(self):
        # Assert that we're defaulting to ``SearchResult``.
        sqs = self.sqs.all()
        self.assertTrue(isinstance(sqs[0], SearchResult))

        # Custom class.
        sqs = self.sqs.result_class(MockSearchResult).all()
        self.assertTrue(isinstance(sqs[0], MockSearchResult))

        # Reset to default.
        sqs = self.sqs.result_class(None).all()
        self.assertTrue(isinstance(sqs[0], SearchResult))
Exemplo n.º 28
0
def plus_search(tags, search, search_types, order=None, in_group=None, extra_filter=None):
    items = get_resources_for_tag_intersection(tags)
    q = None
    for typ, info in search_types:
        if info[0]:
            typ_items = Q(**info[0])
            if info[1]:
                typ_items = typ_items & ~Q(**info[1])
        elif info[1]:
            typ_items = ~Q(**info[1])
        if not q:
            q = typ_items
        else:
            q = q | typ_items

    if extra_filter:
        q = q & Q(**extra_filter)
        
    if q:
        items = items.filter(q)

    included = None
    for obj_class, included_filter in object_type_filters.items():
        objs = obj_class.objects.filter(**included_filter)
        included_search = {'content_type__model':obj_class.__name__.lower(),
                           'object_id__in':objs}
        
        if not included:
            included = Q(**included_search)
        included = included | Q(**included_search)

    items = items.filter(included)

    if in_group:
        # this should be implemented using the code just above and an external search filter arg
        page_ids = WikiPage.objects.filter(in_agent=in_group, stub=False)
        wikipages = Q(**{'content_type__model':'wikipage',
                         'object_id__in':page_ids})
        resource_ids = Resource.objects.filter(in_agent=in_group, stub=False)
        resources = Q(**{'content_type__model':'resource',
                         'object_id__in':resource_ids})

        q = resources | wikipages
        items = items.filter(q)
 
    results_map = {}
    tag_intersection = []
    if search:
        results = RelatedSearchQuerySet().auto_query(search)
        results_map = {}
        if results:
            all_results = results.load_all_queryset(GenericReference, items)
            if order == 'relevance':
                all_results = all_results.load_all()  # this bit is quite evil and makes things really inefficient for large searches
                                              # a better approach would be to get all the ids directly from the fulltext index and use them as a filter for GenericReferences 
                results_map['All'] = [item.object for item in all_results]   #we really really shouldn't do this 
                items_len = len(results_map['All'])
            else:
                search_results = [result.pk for result in all_results]
                results_map['All'] = items.filter(id__in=search_results).order_by(order)
                items_len = results_map['All'].count()
        else:
            results_map = {'All':EmptySearchQuerySet()}
            items_len = 0
    else:
        if items:
            items = items.order_by('creator')
            items_len = items.count()
            results_map['All'] = items
        else:
            items_len = 0
            results_map = {'All':EmptySearchQuerySet()}            

    if order == 'modified':
        results_map['All'] = results_map['All'].order_by('-' + order)
    elif order == 'display_name':
        results_map['All'] = results_map['All'].order_by(order)        

    if 'All' in results_map:
        tag_intersection = get_intersecting_tags(results_map['All'], n=15)

        if len(search_types) > 1:
            for typ, info in search_types:
                if info[0]:
                    typ_items = items.filter(**info[0])
                if info[1]:
                    typ_items = items.exclude(**info[1])
                if search and results and order == 'relevance':
                        # why do this again when we could just separate results using python
                        typ_items = all_results.load_all_queryset(GenericReference, typ_items)
                        typ_items = [item.object for item in typ_items] #we really really shouldn't do this
                if typ_items:
                    results_map[typ] = typ_items
    else:
        results_map = {'All':EmptySearchQuerySet()}

    search_type_data = []
    for typ, data in search_types:
        if results_map.has_key(typ):
            try:
                type_len = results_map[typ].count()
            except TypeError:
                type_len = len(results_map[typ])
            
            search_type_data.append((typ, data[2], results_map[typ], type_len))

    return {'All':results_map['All'], 'items_len':items_len, 'search_types':search_type_data, 'tag_intersection':tag_intersection}
Exemplo n.º 29
0
 def filter_queryset(self, request, queryset, view):
     return self._filter_queryset(request, queryset, view,
                                  RelatedSearchQuerySet())
Exemplo n.º 30
0
def search(request):
    """ Returns messages corresponding to a query """
    mlist_fqdn = request.GET.get("mlist")
    if mlist_fqdn is None:
        mlist = None
    else:
        try:
            mlist = MailingList.objects.get(name=mlist_fqdn)
        except MailingList.DoesNotExist:
            raise Http404("No archived mailing-list by that name.")
        if not is_mlist_authorized(request, mlist):
            return render(request, "hyperkitty/errors/private.html", {
                            "mlist": mlist,
                          }, status=403)


    query = ''
    results = EmptySearchQuerySet()
    sqs = RelatedSearchQuerySet()

    # Remove private non-subscribed lists
    if mlist is not None:
        sqs = sqs.filter(mailinglist__exact=mlist.name)
    else:
        excluded_mlists = MailingList.objects.filter(
            archive_policy=ArchivePolicy.private.value)
        if request.user.is_authenticated():
            subscriptions = request.user.hyperkitty_profile.get_subscriptions()
            excluded_mlists = excluded_mlists.exclude(
                name__in=subscriptions.keys())
        excluded_mlists = excluded_mlists.values_list("name", flat=True)
        sqs = sqs.exclude(mailinglist__in=excluded_mlists)

    # Sorting
    sort_mode = request.GET.get('sort')
    if sort_mode == "date-asc":
        sqs = sqs.order_by("date")
    elif sort_mode == "date-desc":
        sqs = sqs.order_by("-date")

    # Handle data
    if request.GET.get('q'):
        form = SearchForm(
            request.GET, searchqueryset=sqs, load_all=True)
        if form.is_valid():
            query = form.cleaned_data['q']
            results = form.search()
    else:
        form = SearchForm(searchqueryset=sqs, load_all=True)

    messages = paginate(results, page_num=request.GET.get('page'))
    for message in messages:
        if request.user.is_authenticated():
            message.object.myvote = message.object.votes.filter(user=request.user).first()
        else:
            message.object.myvote = None


    context = {
        'mlist' : mlist,
        'form': form,
        'messages': messages,
        'query': query,
        'sort_mode': sort_mode,
        'suggestion': None,
    }
    if results.query.backend.include_spelling:
        context['suggestion'] = form.get_suggestion()

    return render(request, "hyperkitty/search_results.html", context)
Exemplo n.º 31
0
def plus_search(tags,
                search,
                search_types,
                order=None,
                in_group=None,
                extra_filter=None):
    items = get_resources_for_tag_intersection(tags)
    q = None
    for typ, info in search_types:
        if info[0]:
            typ_items = Q(**info[0])
            if info[1]:
                typ_items = typ_items & ~Q(**info[1])
        elif info[1]:
            typ_items = ~Q(**info[1])
        if not q:
            q = typ_items
        else:
            q = q | typ_items

    if extra_filter:
        q = q & Q(**extra_filter)

    if q:
        items = items.filter(q)

    included = None
    for obj_class, included_filter in object_type_filters.items():
        objs = obj_class.objects.filter(**included_filter)
        included_search = {
            'content_type__model': obj_class.__name__.lower(),
            'object_id__in': objs
        }

        if not included:
            included = Q(**included_search)
        included = included | Q(**included_search)

    items = items.filter(included)

    if in_group:
        # this should be implemented using the code just above and an external search filter arg
        page_ids = WikiPage.objects.filter(in_agent=in_group, stub=False)
        wikipages = Q(**{
            'content_type__model': 'wikipage',
            'object_id__in': page_ids
        })
        resource_ids = Resource.objects.filter(in_agent=in_group, stub=False)
        resources = Q(**{
            'content_type__model': 'resource',
            'object_id__in': resource_ids
        })

        q = resources | wikipages
        items = items.filter(q)

    results_map = {}
    tag_intersection = []
    if search:
        results = RelatedSearchQuerySet().auto_query(search)
        results_map = {}
        if results:
            all_results = results.load_all_queryset(GenericReference, items)
            if order == 'relevance':
                all_results = all_results.load_all(
                )  # this bit is quite evil and makes things really inefficient for large searches
                # a better approach would be to get all the ids directly from the fulltext index and use them as a filter for GenericReferences
                results_map['All'] = [item.object for item in all_results
                                      ]  #we really really shouldn't do this
                items_len = len(results_map['All'])
            else:
                search_results = [result.pk for result in all_results]
                results_map['All'] = items.filter(
                    id__in=search_results).order_by(order)
                items_len = results_map['All'].count()
        else:
            results_map = {'All': EmptySearchQuerySet()}
            items_len = 0
    else:
        if items:
            items = items.order_by('creator')
            items_len = items.count()
            results_map['All'] = items
        else:
            items_len = 0
            results_map = {'All': EmptySearchQuerySet()}

    if order == 'modified':
        results_map['All'] = results_map['All'].order_by('-' + order)
    elif order == 'display_name':
        results_map['All'] = results_map['All'].order_by(order)

    if 'All' in results_map:
        tag_intersection = get_intersecting_tags(results_map['All'], n=15)

        if len(search_types) > 1:
            for typ, info in search_types:
                if info[0]:
                    typ_items = items.filter(**info[0])
                if info[1]:
                    typ_items = items.exclude(**info[1])
                if search and results and order == 'relevance':
                    # why do this again when we could just separate results using python
                    typ_items = all_results.load_all_queryset(
                        GenericReference, typ_items)
                    typ_items = [item.object for item in typ_items
                                 ]  #we really really shouldn't do this
                if typ_items:
                    results_map[typ] = typ_items
    else:
        results_map = {'All': EmptySearchQuerySet()}

    search_type_data = []
    for typ, data in search_types:
        if results_map.has_key(typ):
            try:
                type_len = results_map[typ].count()
            except TypeError:
                type_len = len(results_map[typ])

            search_type_data.append((typ, data[2], results_map[typ], type_len))

    return {
        'All': results_map['All'],
        'items_len': items_len,
        'search_types': search_type_data,
        'tag_intersection': tag_intersection
    }
Exemplo n.º 32
0
 def test_empty(self):
     self.assertEqual(len(DataConcept.objects.published()), 12)
     self.assertEqual(RelatedSearchQuerySet().models(DataConcept).count(),
                      12)
     self.assertEqual(len(DataConcept.objects.search('')), 0)
Exemplo n.º 33
0
    def get_context_data(self, **kwargs):
        context = super(SAQuestionIndex, self).get_context_data(**kwargs)

        context['ministers'] = []
        ministers = Section.objects.filter(parent__heading='Questions')
        ministers = sorted(ministers, key=questions_section_sort_key)
        for minister in ministers:
            context['ministers'].append({
                'title':
                minister.title.replace('Questions asked to the ', ''),
                'slug':
                minister.slug
            })

        context['orderby'] = 'recentanswers'
        context['minister'] = 'all'
        context['q'] = ''

        #set filter values
        for key in ('orderby', 'minister', 'q'):
            if key in self.request.GET:
                context[key] = self.request.GET[key]

        if not context['orderby'] in ['recentquestions', 'recentanswers']:
            context['orderby'] = 'recentquestions'

        search_result_sections = []

        if context['q'] != '':
            #using a RelatedSearchQuerySet seems to result in fewer
            #queries, although the same results can be achieved with a
            #SearchQuerySet
            query = RelatedSearchQuerySet().models(Speech)
            query = query.filter(
                tags__name__in=['question', 'answer'],
                content=AutoQuery(context['q']),
            ).load_all()

            all_speeches = Speech.objects.all().filter(
                tags__name__in=['question', 'answer'])

            if context['minister'] != 'all':
                all_speeches = all_speeches.filter(
                    section__parent__slug=context['minister'])

            query = query.load_all_queryset(Speech, all_speeches)

            for result in query:
                search_result_sections.append(result.object.section.id)

        sections = Section \
            .objects \
            .filter(
                parent__parent__heading='Questions'
            ) \
            .select_related('parent') \
            .prefetch_related('speech_set') \
            .annotate(
                earliest_date=Min('speech__start_date'),
                smallest_id=Min('speech__id'),
                number_speeches=Count('speech'),
                latest_date=Max('speech__start_date'),
            )

        if context['minister'] != 'all':
            sections = sections.filter(parent__slug=context['minister'])

        if len(search_result_sections) > 0:
            sections = sections.filter(id__in=search_result_sections)

        if context['orderby'] == 'recentanswers':
            sections = sections.filter(number_speeches__gt=1).order_by(
                '-latest_date', '-smallest_id')
        else:
            sections = sections.order_by('-earliest_date', '-smallest_id')

        paginator = Paginator(sections, 10)
        page = self.request.GET.get('page')

        try:
            sections = paginator.page(page)
        except PageNotAnInteger:
            sections = paginator.page(1)
        except EmptyPage:
            sections = paginator.page(paginator.num_pages)

        context['paginator'] = sections

        #format questions and answers for the template
        questions = []
        for section in sections:
            question = section.speech_set.all()[0]
            question.questionto = section.parent.heading.replace(
                'Questions asked to the ', '')
            question.questionto_slug = section.parent.slug

            #assume if there is more than one speech that the question is answered
            if len(section.speech_set.all()) > 1:
                question.answer = section \
                    .speech_set \
                    .all()[len(section.speech_set.all()) - 1]

                #extract the actual reply from the reply text (replies
                #often include the original question and other text,
                #such as a letterhead)
                text = question.answer.text.split('REPLY')
                if len(text) > 1:
                    question.answer.text = text[len(text) - 1]
                    if question.answer.text[0] == ':':
                        question.answer.text = question.answer.text[1:]

            questions.append(question)

        context['speeches'] = questions

        return context
Exemplo n.º 34
0
                    "events_recording.event_id = events_event. ID AND "
                    "events_recording.media_file_id = multimedia_mediafile. ID AND "
                    " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'"
                    " GROUP BY events_event.id",
                    'audio_count':
                    "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE "
                    "events_recording.event_id = events_event. ID AND "
                    "events_recording.media_file_id = multimedia_mediafile. ID AND "
                    " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='audio'"
                    " GROUP BY events_event.id",
                }))
        return sqs


event_search = EventSearchView(form_class=EventSearchForm,
                               searchqueryset=RelatedSearchQuerySet())


def annotate_events(events):
    return events.annotate(product_count=Count('products')).extra(
        select={
            'video_count':
            "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE "
            "events_recording.event_id = events_event. ID AND "
            "events_recording.media_file_id = multimedia_mediafile. ID AND "
            " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'"
            " GROUP BY events_event.id",
            'audio_count':
            "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE "
            "events_recording.event_id = events_event. ID AND "
            "events_recording.media_file_id = multimedia_mediafile. ID AND "
Exemplo n.º 35
0
class LiveSolrRelatedSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the RelatedSearchQuerySet."""

    fixtures = ["solr_bulk_data.json"]

    def setUp(self):
        super(LiveSolrRelatedSearchQuerySetTestCase, self).setUp()

        # With the models registered, you get the proper bits.
        import haystack
        from haystack.sites import SearchSite

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_site = haystack.site
        test_site = SearchSite()
        test_site.register(MockModel)
        haystack.site = test_site

        self.rsqs = RelatedSearchQuerySet()

        # Force indexing of the content.
        for mock in MockModel.objects.all():
            mock.save()

    def tearDown(self):
        # Restore.
        import haystack

        haystack.site = self.old_site
        settings.DEBUG = self.old_debug
        super(LiveSolrRelatedSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.rsqs.load_all()
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assert_(len(sqs) > 0)
        self.assertEqual(
            sqs[0].object.foo,
            u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:",
        )

    def test_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(2, 24))

    def test_iter(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 4)

    def test_slice(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(backends.queries), 3)

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(backends.queries), 4)

    def test_manual_iter(self):
        results = self.rsqs.all()

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 4)

    def test_fill_cache(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(backends.queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(backends.queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(backends.queries), 2)

    def test_cache_is_full(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(backends.queries), 5)
Exemplo n.º 36
0
def search(request):
    """ Returns messages corresponding to a query """
    mlist_fqdn = request.GET.get("mlist")
    if mlist_fqdn is None:
        mlist = None
    else:
        try:
            mlist = MailingList.objects.get(name=mlist_fqdn)
        except MailingList.DoesNotExist:
            raise Http404("No archived mailing-list by that name.")
        if not is_mlist_authorized(request, mlist):
            return render(request,
                          "hyperkitty/errors/private.html", {
                              "mlist": mlist,
                          },
                          status=403)
    query = ''
    results = EmptySearchQuerySet()
    sqs = RelatedSearchQuerySet()

    # Remove private non-subscribed lists
    if mlist is not None:
        sqs = sqs.filter(mailinglist__exact=mlist.name)
    else:
        excluded_mlists = MailingList.objects.filter(
            archive_policy=ArchivePolicy.private.value)
        if request.user.is_authenticated:
            subscriptions = get_subscriptions(request.user)
            excluded_mlists = excluded_mlists.exclude(
                list_id__in=subscriptions.keys())
        excluded_mlists = excluded_mlists.values_list("name", flat=True)
        sqs = sqs.exclude(mailinglist__in=excluded_mlists)

    # Sorting
    sort_mode = request.GET.get('sort')
    if sort_mode == "date-asc":
        sqs = sqs.order_by("date")
    elif sort_mode == "date-desc":
        sqs = sqs.order_by("-date")

    # Handle data
    if request.GET.get('q'):
        form = SearchForm(request.GET, searchqueryset=sqs, load_all=True)
        if form.is_valid():
            query = form.cleaned_data['q']
            results = form.search()
    else:
        form = SearchForm(searchqueryset=sqs, load_all=True)

    try:
        emails = paginate(
            results,
            request.GET.get('page'),
            request.GET.get('count'),
        )
    except Exception as e:
        backend = settings.HAYSTACK_CONNECTIONS[DEFAULT_ALIAS]["ENGINE"]
        if backend == "haystack.backends.whoosh_backend.WhooshEngine":
            from whoosh.qparser.common import QueryParserError
            search_exception = QueryParserError
        if backend == "xapian_backend.XapianEngine":
            from xapian import QueryParserError
            search_exception = QueryParserError
        if not isinstance(e, search_exception):
            raise
        emails = paginate([])
        form.add_error(
            "q",
            ValidationError(
                _('Parsing error: %(error)s'),
                params={"error": e},
                code="parse",
            ))
    for email in emails:
        if request.user.is_authenticated:
            email.object.myvote = email.object.votes.filter(
                user=request.user).first()
        else:
            email.object.myvote = None

    context = {
        'mlist': mlist,
        'form': form,
        'emails': emails,
        'query': query,
        'sort_mode': sort_mode,
        'suggestion': None,
    }
    if results.query.backend.include_spelling:
        context['suggestion'] = form.get_suggestion()

    return render(request, "hyperkitty/search_results.html", context)
Exemplo n.º 37
0
 def __init__(self, *args, **kwargs):
     if kwargs.get('searchqueryset') is None:
         kwargs['searchqueryset'] = RelatedSearchQuerySet()
     super(SpeechForm, self).__init__(*args, **kwargs)
Exemplo n.º 38
0
 def get_query(self, search_term):
     return RelatedSearchQuerySet().filter(content=search_term)
Exemplo n.º 39
0
 def build_form(self, *args, **kwargs):
     self.searchqueryset = RelatedSearchQuerySet()
     if self.request.GET.get('sort') == 'newest':
         self.searchqueryset = self.searchqueryset.order_by('-start_date')
     return super(CustomSearchView, self).build_form(*args, **kwargs)
Exemplo n.º 40
0
 def __init__(self, *args, **kwargs):
     if not kwargs.get('searchqueryset', None):
         kwargs['searchqueryset'] = RelatedSearchQuerySet()
     super(ProductPredictiveSearchForm, self).__init__(*args, **kwargs)
Exemplo n.º 41
0
    def get_context_data(self, **kwargs):
        context = super(SAQuestionIndex, self).get_context_data(**kwargs)

        context['ministers'] = []
        ministers = Section.objects.filter(parent__heading='Questions')
        ministers = sorted(ministers, key=questions_section_sort_key)
        for minister in ministers:
            context['ministers'].append({
                'title': minister.title.replace('Questions asked to the ', ''),
                'slug': minister.slug
            })

        context['orderby'] = 'recentanswers'
        context['minister'] = 'all'
        context['q'] = ''

        #set filter values
        for key in ('orderby', 'minister', 'q'):
            if key in self.request.GET:
                context[key] = self.request.GET[key]

        if not context['orderby'] in ['recentquestions', 'recentanswers']:
            context['orderby'] = 'recentquestions'

        search_result_sections = []

        if context['q'] != '':
            #using a RelatedSearchQuerySet seems to result in fewer
            #queries, although the same results can be achieved with a
            #SearchQuerySet
            query = RelatedSearchQuerySet().models(Speech)
            query = query.filter(
                tags__name__in=['question', 'answer'],
                content=AutoQuery(context['q']),
            ).load_all()

            all_speeches = Speech.objects.all().filter(
                tags__name__in=['question', 'answer'])

            if context['minister'] != 'all':
                all_speeches = all_speeches.filter(
                    section__parent__slug=context['minister']
                )

            query = query.load_all_queryset(Speech, all_speeches)

            for result in query:
                search_result_sections.append(result.object.section.id)

        sections = Section \
            .objects \
            .filter(
                parent__parent__heading='Questions'
            ) \
            .select_related('parent__heading') \
            .prefetch_related('speech_set') \
            .annotate(
                earliest_date=Min('speech__start_date'),
                smallest_id=Min('speech__id'),
                number_speeches=Count('speech'),
                latest_date=Max('speech__start_date'),
            )

        if context['minister'] != 'all':
            sections = sections.filter(parent__slug=context['minister'])

        if len(search_result_sections) > 0:
            sections = sections.filter(id__in=search_result_sections)

        if context['orderby'] == 'recentanswers':
            sections = sections.filter(number_speeches__gt=1).order_by(
                '-latest_date',
                '-smallest_id'
            )
        else:
            sections = sections.order_by(
                '-earliest_date',
                '-smallest_id'
            )

        paginator = Paginator(sections, 10)
        page = self.request.GET.get('page')

        try:
            sections = paginator.page(page)
        except PageNotAnInteger:
            sections = paginator.page(1)
        except EmptyPage:
            sections = paginator.page(paginator.num_pages)

        context['paginator'] = sections

        #format questions and answers for the template
        questions = []
        for section in sections:
            question = section.speech_set.all()[0]
            question.questionto = section.parent.heading.replace(
                'Questions asked to the ', '')
            question.questionto_slug = section.parent.slug

            #assume if there is more than one speech that the question is answered
            if len(section.speech_set.all()) > 1:
                question.answer = section \
                    .speech_set \
                    .all()[len(section.speech_set.all()) - 1]

                #extract the actual reply from the reply text (replies
                #often include the original question and other text,
                #such as a letterhead)
                text = question.answer.text.split('REPLY')
                if len(text) > 1:
                    question.answer.text = text[len(text) - 1]
                    if question.answer.text[0] == ':':
                        question.answer.text = question.answer.text[1:]

            questions.append(question)

        context['speeches'] = questions

        return context
class LiveElasticsearchSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['bulk_data.json']

    def setUp(self):
        super(LiveElasticsearchSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_elasticsearch_index()

            # Force indexing of the content.
            self.smmi.update()

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        settings.DEBUG = self.old_debug
        super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.sqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')

    def test_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        results = sorted([int(result.pk) for result in sqs])
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(int(results[21].pk), 18)
        self.assertEqual(len(connections['default'].queries), 1)

    def test_count(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(len(sqs), 23)
        self.assertEqual(sqs.count(), 23)
        # Should only execute one query to count the length of the result set.
        self.assertEqual(len(connections['default'].queries), 1)

    def test_manual_iter(self):
        results = self.sqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
        self.assertEqual(len(connections['default'].queries), 3)

    def test_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 3)

    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)')

    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)')

    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.
        sqs = self.sqs.auto_query('"pants:rule"')
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">')
        self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"')
        self.assertEqual(len(sqs), 0)

    # Regressions

    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)

        id_counts = {}

        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1

        for key, value in id_counts.items():
            if value > 1:
                self.fail("Result with id '%s' seen more than once in the results." % key)

    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text:index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]

        for pk in page_2:
            if pk in page_1:
                self.fail("Result with id '%s' seen more than once in the results." % pk)

    # RelatedSearchQuerySet Tests

    def test_related_load_all(self):
        sqs = self.rsqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')

    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual(sorted([obj.object.id for obj in sqs]), range(2, 24))

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], [12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20])
        self.assertEqual([obj.object.id for obj in sqs[10:20]], [13, 18, 20])

    def test_related_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
        self.assertEqual(len(connections['default'].queries), 3)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 18)
        self.assertEqual(len(connections['default'].queries), 4)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[20:30]], [13, 18, 20])
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_manual_iter(self):
        results = self.rsqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = sorted([int(result.pk) for result in results._manual_iter()])
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_related_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 5)

    def test_quotes_regression(self):
        sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
        # Should not have empty terms.
        self.assertEqual(sqs.query.build_query(), u"44\xb048'40''N 20\xb028'32''E")
        # Should not cause Elasticsearch to 500.
        self.assertEqual(sqs.count(), 0)

        sqs = self.sqs.auto_query('blazing')
        self.assertEqual(sqs.query.build_query(), u'blazing')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'blazing saddles')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'\\"blazing saddles')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'"blazing saddles"')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles"')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'saddles"')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing \'saddles"')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles"')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles" \'')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles" \'\\"')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel')
        self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
        self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel brooks')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles" brooks')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
        self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles" \\"brooks')
        self.assertEqual(sqs.count(), 0)

    def test_result_class(self):
        # Assert that we're defaulting to ``SearchResult``.
        sqs = self.sqs.all()
        self.assertTrue(isinstance(sqs[0], SearchResult))

        # Custom class.
        sqs = self.sqs.result_class(MockSearchResult).all()
        self.assertTrue(isinstance(sqs[0], MockSearchResult))

        # Reset to default.
        sqs = self.sqs.result_class(None).all()
        self.assertTrue(isinstance(sqs[0], SearchResult))
Exemplo n.º 43
0
def search(request):
    """ Returns messages corresponding to a query """
    mlist_fqdn = request.GET.get("mlist")
    if mlist_fqdn is None:
        mlist = None
    else:
        try:
            mlist = MailingList.objects.get(name=mlist_fqdn)
        except MailingList.DoesNotExist:
            raise Http404("No archived mailing-list by that name.")
        if not is_mlist_authorized(request, mlist):
            return render(request, "hyperkitty/errors/private.html", {
                            "mlist": mlist,
                          }, status=403)


    query = ''
    results = EmptySearchQuerySet()
    sqs = RelatedSearchQuerySet()

    # Remove private non-subscribed lists
    if mlist is not None:
        sqs = sqs.filter(mailinglist__exact=mlist.name)
    else:
        excluded_mlists = MailingList.objects.filter(
            archive_policy=ArchivePolicy.private.value)
        if request.user.is_authenticated():
            subscriptions = request.user.hyperkitty_profile.get_subscriptions()
            excluded_mlists = excluded_mlists.exclude(
                list_id__in=subscriptions.keys())
        excluded_mlists = excluded_mlists.values_list("name", flat=True)
        sqs = sqs.exclude(mailinglist__in=excluded_mlists)

    # Sorting
    sort_mode = request.GET.get('sort')
    if sort_mode == "date-asc":
        sqs = sqs.order_by("date")
    elif sort_mode == "date-desc":
        sqs = sqs.order_by("-date")

    # Handle data
    if request.GET.get('q'):
        form = SearchForm(
            request.GET, searchqueryset=sqs, load_all=True)
        if form.is_valid():
            query = form.cleaned_data['q']
            results = form.search()
    else:
        form = SearchForm(searchqueryset=sqs, load_all=True)

    emails = paginate(results, page_num=request.GET.get('page'))
    for email in emails:
        if request.user.is_authenticated():
            email.object.myvote = email.object.votes.filter(user=request.user).first()
        else:
            email.object.myvote = None


    context = {
        'mlist' : mlist,
        'form': form,
        'emails': emails,
        'query': query,
        'sort_mode': sort_mode,
        'suggestion': None,
    }
    if results.query.backend.include_spelling:
        context['suggestion'] = form.get_suggestion()

    return render(request, "hyperkitty/search_results.html", context)
Exemplo n.º 44
0
class LiveElasticsearchSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['bulk_data.json']

    def setUp(self):
        super(LiveElasticsearchSearchQuerySetTestCase, self).setUp()

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_ui = connections['default'].get_unified_index()
        self.ui = UnifiedIndex()
        self.smmi = ElasticsearchMockSearchIndex()
        self.ui.build(indexes=[self.smmi])
        connections['default']._index = self.ui

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_elasticsearch_index()

            # Force indexing of the content.
            self.smmi.update()

    def tearDown(self):
        # Restore.
        connections['default']._index = self.old_ui
        settings.DEBUG = self.old_debug
        super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.sqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')

    def test_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        results = sorted([int(result.pk) for result in sqs])
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 3)

    def test_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
        self.assertEqual(len(connections['default'].queries), 1)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(int(results[21].pk), 18)
        self.assertEqual(len(connections['default'].queries), 1)

    def test_count(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.sqs.all()
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(len(sqs), 23)
        self.assertEqual(sqs.count(), 23)
        # Should only execute one query to count the length of the result set.
        self.assertEqual(len(connections['default'].queries), 1)

    def test_manual_iter(self):
        results = self.sqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
        self.assertEqual(len(connections['default'].queries), 3)

    def test_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 3)

    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(sqs.query.build_query(), u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))')

    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4

        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))')

    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.
        sqs = self.sqs.auto_query('"pants:rule"')
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">')
        self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")')
        self.assertEqual(len(sqs), 0)

    # Regressions

    @unittest.expectedFailure
    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)

        id_counts = {}

        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1

        for key, value in id_counts.items():
            if value > 1:
                self.fail("Result with id '%s' seen more than once in the results." % key)

    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text:index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]

        for pk in page_2:
            if pk in page_1:
                self.fail("Result with id '%s' seen more than once in the results." % pk)

    # RelatedSearchQuerySet Tests

    def test_related_load_all(self):
        sqs = self.rsqs.load_all()
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertTrue(len(sqs) > 0)
        self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')

    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual(sorted([obj.object.id for obj in sqs]), range(2, 24))

        sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
        self.assertTrue(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], [12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20])
        self.assertEqual([obj.object.id for obj in sqs[10:20]], [13, 18, 20])

    def test_related_iter(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_slice(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
        self.assertEqual(len(connections['default'].queries), 3)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 18)
        self.assertEqual(len(connections['default'].queries), 4)

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[20:30]], [13, 18, 20])
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_manual_iter(self):
        results = self.rsqs.all()

        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = sorted([int(result.pk) for result in results._manual_iter()])
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(connections['default'].queries), 4)

    def test_related_fill_cache(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(connections['default'].queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
        self.assertEqual(len(connections['default'].queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
        self.assertEqual(len(connections['default'].queries), 2)

    def test_related_cache_is_full(self):
        reset_search_queries()
        self.assertEqual(len(connections['default'].queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(connections['default'].queries), 5)

    def test_quotes_regression(self):
        sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
        # Should not have empty terms.
        self.assertEqual(sqs.query.build_query(), u"(44\xb048'40''N 20\xb028'32''E)")
        # Should not cause Elasticsearch to 500.
        self.assertEqual(sqs.count(), 0)

        sqs = self.sqs.auto_query('blazing')
        self.assertEqual(sqs.query.build_query(), u'(blazing)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(blazing saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles')
        self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \')')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \'\\")')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
        self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" brooks)')
        self.assertEqual(sqs.count(), 0)
        sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
        self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" \\"brooks)')
        self.assertEqual(sqs.count(), 0)

    def test_query_generation(self):
        sqs = self.sqs.filter(SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")))
        self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))")

    def test_result_class(self):
        # Assert that we're defaulting to ``SearchResult``.
        sqs = self.sqs.all()
        self.assertTrue(isinstance(sqs[0], SearchResult))

        # Custom class.
        sqs = self.sqs.result_class(MockSearchResult).all()
        self.assertTrue(isinstance(sqs[0], MockSearchResult))

        # Reset to default.
        sqs = self.sqs.result_class(None).all()
        self.assertTrue(isinstance(sqs[0], SearchResult))
Exemplo n.º 45
0
 def build_form(self, *args, **kwargs):
     self.searchqueryset = RelatedSearchQuerySet()
     if self.request.GET.get("sort") == "newest":
         self.searchqueryset = self.searchqueryset.order_by("-start_date")
     return super(CustomSearchView, self).build_form(*args, **kwargs)
Exemplo n.º 46
0
handler500 = "pinax.views.server_error"


urlpatterns = patterns("",
    url(r"^", include("pins.urls")),
    url(r"^admin/", include(admin.site.urls)),
    url(r'^admin_tools/', include('admin_tools.urls')),
    url(r"^about/", include("about.urls")),
    url(r"^account/", include("account.urls")),
    url(r"^profile/", include("profiles.urls")),
    url(r'^comments/', include('django.contrib.comments.urls')),
    url('^activity/', include('actstream.urls')),
    url(r'^abuse/', include('abuse_reports.urls')),
    url(r'^invite/', include('invite_friends.urls')),
    url(r'^auth/', include('social_auth.urls')),
    url(r'^cms/', include('cms.urls')),
)

urlpatterns = urlpatterns + patterns('haystack.views',
    url(r'^search/$', SearchView(searchqueryset=RelatedSearchQuerySet()), name='haystack_search'),
)


if settings.SERVE_MEDIA:
    urlpatterns += patterns("",
        url(r"", include("staticfiles.urls"),),
        url(r'^site_media/media/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': settings.MEDIA_ROOT}),
        url(r'^site_media/static/(?P<path>.*)$', 'django.views.static.serve',
        {'document_root': settings.STATIC_ROOT}),
    )
Exemplo n.º 47
0
class LiveSolrSearchQuerySetTestCase(TestCase):
    """Used to test actual implementation details of the SearchQuerySet."""
    fixtures = ['bulk_data.json']

    def setUp(self):
        super(LiveSolrSearchQuerySetTestCase, self).setUp()

        # With the models registered, you get the proper bits.
        import haystack
        from haystack.sites import SearchSite

        # Stow.
        self.old_debug = settings.DEBUG
        settings.DEBUG = True
        self.old_site = haystack.site
        test_site = SearchSite()
        test_site.register(MockModel, SolrMockModelSearchIndex)
        haystack.site = test_site

        self.sqs = SearchQuerySet()
        self.rsqs = RelatedSearchQuerySet()

        # Ugly but not constantly reindexing saves us almost 50% runtime.
        global lssqstc_all_loaded

        if lssqstc_all_loaded is None:
            print 'Reloading data...'
            lssqstc_all_loaded = True

            # Wipe it clean.
            clear_solr_index()

            # Force indexing of the content.
            mockmodel_index = test_site.get_index(MockModel)
            mockmodel_index.update()

    def tearDown(self):
        # Restore.
        import haystack
        haystack.site = self.old_site
        settings.DEBUG = self.old_debug
        super(LiveSolrSearchQuerySetTestCase, self).tearDown()

    def test_load_all(self):
        sqs = self.sqs.load_all()
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assert_(len(sqs) > 0)
        self.assertEqual(
            sqs[0].object.foo,
            u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
        )

    def test_iter(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        sqs = self.sqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 3)

    def test_slice(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.sqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]],
                         [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(backends.queries), 1)

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.sqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(backends.queries), 1)

    def test_count(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        sqs = self.sqs.all()
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(sqs.count(), 23)
        self.assertEqual(len(sqs), 23)
        self.assertEqual(sqs.count(), 23)
        # Should only execute one query to count the length of the result set.
        self.assertEqual(len(backends.queries), 1)

    def test_manual_iter(self):
        results = self.sqs.all()

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 3)

    def test_fill_cache(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.sqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(backends.queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 10)
        self.assertEqual(len(backends.queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 20)
        self.assertEqual(len(backends.queries), 2)

    def test_cache_is_full(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        self.assertEqual(self.sqs._cache_is_full(), False)
        results = self.sqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(backends.queries), 3)

    def test___and__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 & sqs2

        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(
            title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar')
        sqs = sqs3 & sqs4

        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 3)
        self.assertEqual(sqs.query.build_query(),
                         u'(NOT (title:moof) AND (foo OR baz) AND bar)')

    def test___or__(self):
        sqs1 = self.sqs.filter(content='foo')
        sqs2 = self.sqs.filter(content='bar')
        sqs = sqs1 | sqs2

        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')

        # Now for something more complex...
        sqs3 = self.sqs.exclude(
            title='moof').filter(SQ(content='foo') | SQ(content='baz'))
        sqs4 = self.sqs.filter(content='bar').models(MockModel)
        sqs = sqs3 | sqs4

        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs.query.query_filter), 2)
        self.assertEqual(sqs.query.build_query(),
                         u'((NOT (title:moof) AND (foo OR baz)) OR bar)')

    def test_auto_query(self):
        # Ensure bits in exact matches get escaped properly as well.
        # This will break horrifically if escaping isn't working.
        sqs = self.sqs.auto_query('"pants:rule"')
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(repr(sqs.query.query_filter),
                         '<SQ: AND content__exact=pants\\:rule>')
        self.assertEqual(sqs.query.build_query(), u'pants\\:rule')
        self.assertEqual(len(sqs), 0)

    # Regressions

    def test_regression_proper_start_offsets(self):
        sqs = self.sqs.filter(text='index')
        self.assertNotEqual(sqs.count(), 0)

        id_counts = {}

        for item in sqs:
            if item.id in id_counts:
                id_counts[item.id] += 1
            else:
                id_counts[item.id] = 1

        for key, value in id_counts.items():
            if value > 1:
                self.fail(
                    "Result with id '%s' seen more than once in the results." %
                    key)

    def test_regression_raw_search_breaks_slicing(self):
        sqs = self.sqs.raw_search('text: index')
        page_1 = [result.pk for result in sqs[0:10]]
        page_2 = [result.pk for result in sqs[10:20]]

        for pk in page_2:
            if pk in page_1:
                self.fail(
                    "Result with id '%s' seen more than once in the results." %
                    pk)

    # RelatedSearchQuerySet Tests

    def test_related_load_all(self):
        sqs = self.rsqs.load_all()
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assert_(len(sqs) > 0)
        self.assertEqual(
            sqs[0].object.foo,
            u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_.  If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class.  This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
        )

    def test_related_load_all_queryset(self):
        sqs = self.rsqs.load_all()
        self.assertEqual(len(sqs._load_all_querysets), 0)

        sqs = sqs.load_all_queryset(MockModel,
                                    MockModel.objects.filter(id__gt=1))
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(2, 24))

        sqs = sqs.load_all_queryset(MockModel,
                                    MockModel.objects.filter(id__gt=10))
        self.assert_(isinstance(sqs, SearchQuerySet))
        self.assertEqual(len(sqs._load_all_querysets), 1)
        self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
        self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])

    def test_related_iter(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        sqs = self.rsqs.all()
        results = [int(result.pk) for result in sqs]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 4)

    def test_related_slice(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[1:11]],
                         [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
        self.assertEqual(len(backends.queries), 3)

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual(int(results[21].pk), 22)
        self.assertEqual(len(backends.queries), 4)

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual([int(result.pk) for result in results[20:30]],
                         [21, 22, 23])
        self.assertEqual(len(backends.queries), 4)

    def test_related_manual_iter(self):
        results = self.rsqs.all()

        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = [int(result.pk) for result in results._manual_iter()]
        self.assertEqual(results, range(1, 24))
        self.assertEqual(len(backends.queries), 4)

    def test_related_fill_cache(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        results = self.rsqs.all()
        self.assertEqual(len(results._result_cache), 0)
        self.assertEqual(len(backends.queries), 0)
        results._fill_cache(0, 10)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 10)
        self.assertEqual(len(backends.queries), 1)
        results._fill_cache(10, 20)
        self.assertEqual(
            len([
                result for result in results._result_cache
                if result is not None
            ]), 20)
        self.assertEqual(len(backends.queries), 2)

    def test_related_cache_is_full(self):
        backends.reset_search_queries()
        self.assertEqual(len(backends.queries), 0)
        self.assertEqual(self.rsqs._cache_is_full(), False)
        results = self.rsqs.all()
        fire_the_iterator_and_fill_cache = [result for result in results]
        self.assertEqual(results._cache_is_full(), True)
        self.assertEqual(len(backends.queries), 5)
Exemplo n.º 48
0
 def build_form(self, *args, **kwargs):
     self.searchqueryset = RelatedSearchQuerySet()
     if self.request.GET.get('sort') != 'relevance':
         self.searchqueryset = self.searchqueryset.order_by('-start_date')
     return super(CustomSearchView, self).build_form(*args, **kwargs)