def _haystack_search(self, content, queryset, max_results, partial, using): from haystack.query import RelatedSearchQuerySet from haystack.inputs import AutoQuery if content.strip() == '': return RelatedSearchQuerySet().none() # Limit to the model bound to this manager, e.g. DataConcept. # `load_all` ensures a single database hit when loading the objects # that match sqs = RelatedSearchQuerySet().models(self.model).load_all() # If a non-default backend is being used, set which backend is to # be used. if using is not None: sqs = sqs.using(using) if partial: # Autocomplete only works with N-gram fields. sqs = sqs.autocomplete(text_auto=content) else: # Automatically handles advanced search syntax for negations # and quoted strings. sqs = sqs.filter(text=AutoQuery(content)) if queryset is not None: sqs = sqs.load_all_queryset(self.model, queryset) if max_results: return sqs[:max_results] return sqs
def search_products( request, template_name='satchless/search/haystack_predictive/products.html'): form = forms.ProductPredictiveSearchForm(data=request.GET or None) if form.is_valid(): results = form.search() query = form.cleaned_data['q'] else: results = RelatedSearchQuerySet().all() results = results.load_all() results = results.models(Product) query = '' products_per_page = getattr(settings, 'HAYSTACK_PREDICTIVE_PRODUCTS_PER_PAGE', PRODUCTS_PER_PAGE) paginator = Paginator(results, products_per_page) try: page = paginator.page(request.GET.get('page', 1)) except InvalidPage: raise Http404() return direct_to_template(request, template_name, { 'form': form, 'page': page, 'paginator': paginator, 'query': query, })
def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_ui = connections['default'].get_unified_index() self.ui = UnifiedIndex() self.smmi = SolrMockSearchIndex() self.ui.build(indexes=[self.smmi]) connections['default']._index = self.ui self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. self.smmi.update()
def get_queryset(self): query = self.request.GET.get("query", "") order_by = self.request.GET.get("order", "") try: queryset = ( RelatedSearchQuerySet().models(ForumMessage).autocomplete( auto=html.escape(query))) except TypeError: return [] if order_by == "date": queryset = queryset.order_by("-date") queryset = queryset.load_all() queryset = queryset.load_all_queryset( ForumMessage, ForumMessage.objects.all().prefetch_related( "topic__forum__edit_groups").prefetch_related( "topic__forum__view_groups").prefetch_related( "topic__forum__owner_club"), ) # Filter unauthorized responses resp = [] count = 0 max_count = 30 for r in queryset: if count >= max_count: return resp if can_view(r.object, self.request.user) and can_view( r.object.topic, self.request.user): resp.append(r.object) count += 1 return resp
def setUp(self): super(LiveSolrSearchQuerySetTestCase, self).setUp() # With the models registered, you get the proper bits. import haystack from haystack.sites import SearchSite # Stow. self.old_debug = settings.DEBUG settings.DEBUG = True self.old_site = haystack.site test_site = SearchSite() test_site.register(MockModel, SolrMockModelSearchIndex) haystack.site = test_site self.sqs = SearchQuerySet() self.rsqs = RelatedSearchQuerySet() # Ugly but not constantly reindexing saves us almost 50% runtime. global lssqstc_all_loaded if lssqstc_all_loaded is None: print 'Reloading data...' lssqstc_all_loaded = True # Wipe it clean. clear_solr_index() # Force indexing of the content. mockmodel_index = test_site.get_index(MockModel) mockmodel_index.update()
def __init__(self, GET, *args, **kwargs): # Save all GET parameters in case they match facets and not form fields self.filters = GET super(FeedSearchForm, self).__init__(GET, *args, **kwargs) self.searcher = Searcher(model=Trial, facets=my_facet_config, queryset=RelatedSearchQuerySet())
def search(self, query): if not query: return EmptySearchQuerySet() # OPTIMIZE: the number of public projects can increase substantially # causing a really high number of project_ids to be sended to # elasticsearch projects = Project.objects.public_or_collaborate(self.request.user) return RelatedSearchQuerySet()\ .filter(project_id__in=projects.values_list('pk', flat=True))\ .filter(SQ(content__contains=Clean(query)) | SQ(title__contains=Clean(query)))
def get_results(self, request): if not SEARCH_VAR in request.GET: return super(AllChangeList, self).get_results(request) # Note that pagination is 0-based, not 1-based. sqs = RelatedSearchQuerySet().models(self.model).auto_query( request.GET[SEARCH_VAR]).load_all() if not request.user.is_superuser: result = self.model.objects.distinct().filter( Q(owners=request.user) | Q(editor_groups__name__in=request.user.groups.values_list( 'name', flat=True))) else: result = self.model.objects.distinct() if 'storage_object__publication_status__exact' in request.GET: result = result.filter( storage_object__publication_status__exact=request. GET["storage_object__publication_status__exact"]) sqs = sqs.load_all_queryset(self.model, result) paginator = Paginator(sqs, self.list_per_page) # Fixed bug for pagination, count full_result_count full_result_count = 0 for sq in sqs: full_result_count += 1 result_count = paginator.count # Get the number of objects, with admin filters applied. can_show_all = result_count <= list_max_show_all(self) multi_page = result_count > self.list_per_page # Get the list of objects to display on this page. try: result_list = paginator.page(self.page_num + 1).object_list # Grab just the Django models, since that's what everything else is # expecting. result_list = [result.object for result in result_list] except InvalidPage: result_list = () self.result_count = result_count self.full_result_count = full_result_count self.result_list = result_list self.can_show_all = can_show_all self.multi_page = multi_page self.paginator = paginator
def create_queryset(user): # ideally we'd do it this way, and not mention specific models here # (instead doing visibility checking in their respective search_index.py files), # but there's no way to pass a user into serach_index.load_all_queryset. yet. # # return RelatedSearchQuerySet() qs = RelatedSearchQuerySet().load_all_queryset( GroupTopic, GroupTopic.objects.visible(user)) qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user)) qs = qs.load_all_queryset(Event, Event.objects.visible(user)) # TODO: centralize this somewhere? execlist = get_object_or_none(Community, slug='exec') if execlist and execlist.user_is_member(user, True): qs = qs.load_all_queryset(Activity, Activity.objects.all()) else: qs = qs.load_all_queryset(Activity, Activity.objects.none()) return qs
def filter_queryset(self, request, queryset, view): current_user = request.user permissioned_qs = None if current_user.is_staff: permissioned_qs = queryset elif not current_user.is_anonymous(): user_profile = UserProfile.objects.get(user=current_user) permissioned_qs = queryset.filter( Q(parent_id=user_profile.id, parent_type=ContentType.objects.get_for_model(UserProfile)) | Q(parent_id__in=user_profile.organizations, parent_type=ContentType.objects.get_for_model( Organization)) | ~Q(public_access=ACCESS_TYPE_NONE)) else: permissioned_qs = queryset.filter(~Q( public_access=ACCESS_TYPE_NONE)) return super(ConceptContainerPermissionedSearchFilter, self)._filter_queryset(request, permissioned_qs, view, RelatedSearchQuerySet())
def build_form(self, *args, **kwargs): self.searchqueryset = RelatedSearchQuerySet() if self.request.GET.get('sort') == 'newest': self.searchqueryset = self.searchqueryset.order_by('-start_date') return super(CustomSearchView, self).build_form(*args, **kwargs)
def __init__(self, *args, **kwargs): if not kwargs.get('searchqueryset', None): kwargs['searchqueryset'] = RelatedSearchQuerySet() super(ProductPredictiveSearchForm, self).__init__(*args, **kwargs)
def __init__(self, *args, **kwargs): if kwargs.get('searchqueryset') is None: kwargs['searchqueryset'] = RelatedSearchQuerySet() super(SpeechForm, self).__init__(*args, **kwargs)
def get_query(self, search_term): return RelatedSearchQuerySet().filter(content=search_term)
def filter_queryset(self, request, queryset, view): return self._filter_queryset(request, queryset, view, RelatedSearchQuerySet())
def search(request): """ Returns messages corresponding to a query """ mlist_fqdn = request.GET.get("mlist") if mlist_fqdn is None: mlist = None else: try: mlist = MailingList.objects.get(name=mlist_fqdn) except MailingList.DoesNotExist: raise Http404("No archived mailing-list by that name.") if not is_mlist_authorized(request, mlist): return render(request, "hyperkitty/errors/private.html", { "mlist": mlist, }, status=403) query = '' results = EmptySearchQuerySet() sqs = RelatedSearchQuerySet() # Remove private non-subscribed lists if mlist is not None: sqs = sqs.filter(mailinglist__exact=mlist.name) else: excluded_mlists = MailingList.objects.filter( archive_policy=ArchivePolicy.private.value) if request.user.is_authenticated: subscriptions = get_subscriptions(request.user) excluded_mlists = excluded_mlists.exclude( list_id__in=subscriptions.keys()) excluded_mlists = excluded_mlists.values_list("name", flat=True) sqs = sqs.exclude(mailinglist__in=excluded_mlists) # Sorting sort_mode = request.GET.get('sort') if sort_mode == "date-asc": sqs = sqs.order_by("date") elif sort_mode == "date-desc": sqs = sqs.order_by("-date") # Handle data if request.GET.get('q'): form = SearchForm(request.GET, searchqueryset=sqs, load_all=True) if form.is_valid(): query = form.cleaned_data['q'] results = form.search() else: form = SearchForm(searchqueryset=sqs, load_all=True) try: emails = paginate( results, request.GET.get('page'), request.GET.get('count'), ) except Exception as e: backend = settings.HAYSTACK_CONNECTIONS[DEFAULT_ALIAS]["ENGINE"] if backend == "haystack.backends.whoosh_backend.WhooshEngine": from whoosh.qparser.common import QueryParserError search_exception = QueryParserError if backend == "xapian_backend.XapianEngine": from xapian import QueryParserError search_exception = QueryParserError if not isinstance(e, search_exception): raise emails = paginate([]) form.add_error( "q", ValidationError( _('Parsing error: %(error)s'), params={"error": e}, code="parse", )) for email in emails: if request.user.is_authenticated: email.object.myvote = email.object.votes.filter( user=request.user).first() else: email.object.myvote = None context = { 'mlist': mlist, 'form': form, 'emails': emails, 'query': query, 'sort_mode': sort_mode, 'suggestion': None, } if results.query.backend.include_spelling: context['suggestion'] = form.get_suggestion() return render(request, "hyperkitty/search_results.html", context)
handler500 = "pinax.views.server_error" urlpatterns = patterns("", url(r"^", include("pins.urls")), url(r"^admin/", include(admin.site.urls)), url(r'^admin_tools/', include('admin_tools.urls')), url(r"^about/", include("about.urls")), url(r"^account/", include("account.urls")), url(r"^profile/", include("profiles.urls")), url(r'^comments/', include('django.contrib.comments.urls')), url('^activity/', include('actstream.urls')), url(r'^abuse/', include('abuse_reports.urls')), url(r'^invite/', include('invite_friends.urls')), url(r'^auth/', include('social_auth.urls')), url(r'^cms/', include('cms.urls')), ) urlpatterns = urlpatterns + patterns('haystack.views', url(r'^search/$', SearchView(searchqueryset=RelatedSearchQuerySet()), name='haystack_search'), ) if settings.SERVE_MEDIA: urlpatterns += patterns("", url(r"", include("staticfiles.urls"),), url(r'^site_media/media/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.MEDIA_ROOT}), url(r'^site_media/static/(?P<path>.*)$', 'django.views.static.serve', {'document_root': settings.STATIC_ROOT}), )
"events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND " " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'" " GROUP BY events_event.id", 'audio_count': "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE " "events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND " " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='audio'" " GROUP BY events_event.id", })) return sqs event_search = EventSearchView(form_class=EventSearchForm, searchqueryset=RelatedSearchQuerySet()) def annotate_events(events): return events.annotate(product_count=Count('products')).extra( select={ 'video_count': "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE " "events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND " " events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'" " GROUP BY events_event.id", 'audio_count': "SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE " "events_recording.event_id = events_event. ID AND " "events_recording.media_file_id = multimedia_mediafile. ID AND "
def plus_search(tags, search, search_types, order=None, in_group=None, extra_filter=None): items = get_resources_for_tag_intersection(tags) q = None for typ, info in search_types: if info[0]: typ_items = Q(**info[0]) if info[1]: typ_items = typ_items & ~Q(**info[1]) elif info[1]: typ_items = ~Q(**info[1]) if not q: q = typ_items else: q = q | typ_items if extra_filter: q = q & Q(**extra_filter) if q: items = items.filter(q) included = None for obj_class, included_filter in object_type_filters.items(): objs = obj_class.objects.filter(**included_filter) included_search = { 'content_type__model': obj_class.__name__.lower(), 'object_id__in': objs } if not included: included = Q(**included_search) included = included | Q(**included_search) items = items.filter(included) if in_group: # this should be implemented using the code just above and an external search filter arg page_ids = WikiPage.objects.filter(in_agent=in_group, stub=False) wikipages = Q(**{ 'content_type__model': 'wikipage', 'object_id__in': page_ids }) resource_ids = Resource.objects.filter(in_agent=in_group, stub=False) resources = Q(**{ 'content_type__model': 'resource', 'object_id__in': resource_ids }) q = resources | wikipages items = items.filter(q) results_map = {} tag_intersection = [] if search: results = RelatedSearchQuerySet().auto_query(search) results_map = {} if results: all_results = results.load_all_queryset(GenericReference, items) if order == 'relevance': all_results = all_results.load_all( ) # this bit is quite evil and makes things really inefficient for large searches # a better approach would be to get all the ids directly from the fulltext index and use them as a filter for GenericReferences results_map['All'] = [item.object for item in all_results ] #we really really shouldn't do this items_len = len(results_map['All']) else: search_results = [result.pk for result in all_results] results_map['All'] = items.filter( id__in=search_results).order_by(order) items_len = results_map['All'].count() else: results_map = {'All': EmptySearchQuerySet()} items_len = 0 else: if items: items = items.order_by('creator') items_len = items.count() results_map['All'] = items else: items_len = 0 results_map = {'All': EmptySearchQuerySet()} if order == 'modified': results_map['All'] = results_map['All'].order_by('-' + order) elif order == 'display_name': results_map['All'] = results_map['All'].order_by(order) if 'All' in results_map: tag_intersection = get_intersecting_tags(results_map['All'], n=15) if len(search_types) > 1: for typ, info in search_types: if info[0]: typ_items = items.filter(**info[0]) if info[1]: typ_items = items.exclude(**info[1]) if search and results and order == 'relevance': # why do this again when we could just separate results using python typ_items = all_results.load_all_queryset( GenericReference, typ_items) typ_items = [item.object for item in typ_items ] #we really really shouldn't do this if typ_items: results_map[typ] = typ_items else: results_map = {'All': EmptySearchQuerySet()} search_type_data = [] for typ, data in search_types: if results_map.has_key(typ): try: type_len = results_map[typ].count() except TypeError: type_len = len(results_map[typ]) search_type_data.append((typ, data[2], results_map[typ], type_len)) return { 'All': results_map['All'], 'items_len': items_len, 'search_types': search_type_data, 'tag_intersection': tag_intersection }
def test_empty(self): self.assertEqual(len(DataConcept.objects.published()), 12) self.assertEqual(RelatedSearchQuerySet().models(DataConcept).count(), 12) self.assertEqual(len(DataConcept.objects.search('')), 0)
def get_context_data(self, **kwargs): context = super(SAQuestionIndex, self).get_context_data(**kwargs) context['ministers'] = [] ministers = Section.objects.filter(parent__heading='Questions') ministers = sorted(ministers, key=questions_section_sort_key) for minister in ministers: context['ministers'].append({ 'title': minister.title.replace('Questions asked to the ', ''), 'slug': minister.slug }) context['orderby'] = 'recentanswers' context['minister'] = 'all' context['q'] = '' #set filter values for key in ('orderby', 'minister', 'q'): if key in self.request.GET: context[key] = self.request.GET[key] if not context['orderby'] in ['recentquestions', 'recentanswers']: context['orderby'] = 'recentquestions' search_result_sections = [] if context['q'] != '': #using a RelatedSearchQuerySet seems to result in fewer #queries, although the same results can be achieved with a #SearchQuerySet query = RelatedSearchQuerySet().models(Speech) query = query.filter( tags__name__in=['question', 'answer'], content=AutoQuery(context['q']), ).load_all() all_speeches = Speech.objects.all().filter( tags__name__in=['question', 'answer']) if context['minister'] != 'all': all_speeches = all_speeches.filter( section__parent__slug=context['minister']) query = query.load_all_queryset(Speech, all_speeches) for result in query: search_result_sections.append(result.object.section.id) sections = Section \ .objects \ .filter( parent__parent__heading='Questions' ) \ .select_related('parent') \ .prefetch_related('speech_set') \ .annotate( earliest_date=Min('speech__start_date'), smallest_id=Min('speech__id'), number_speeches=Count('speech'), latest_date=Max('speech__start_date'), ) if context['minister'] != 'all': sections = sections.filter(parent__slug=context['minister']) if len(search_result_sections) > 0: sections = sections.filter(id__in=search_result_sections) if context['orderby'] == 'recentanswers': sections = sections.filter(number_speeches__gt=1).order_by( '-latest_date', '-smallest_id') else: sections = sections.order_by('-earliest_date', '-smallest_id') paginator = Paginator(sections, 10) page = self.request.GET.get('page') try: sections = paginator.page(page) except PageNotAnInteger: sections = paginator.page(1) except EmptyPage: sections = paginator.page(paginator.num_pages) context['paginator'] = sections #format questions and answers for the template questions = [] for section in sections: question = section.speech_set.all()[0] question.questionto = section.parent.heading.replace( 'Questions asked to the ', '') question.questionto_slug = section.parent.slug #assume if there is more than one speech that the question is answered if len(section.speech_set.all()) > 1: question.answer = section \ .speech_set \ .all()[len(section.speech_set.all()) - 1] #extract the actual reply from the reply text (replies #often include the original question and other text, #such as a letterhead) text = question.answer.text.split('REPLY') if len(text) > 1: question.answer.text = text[len(text) - 1] if question.answer.text[0] == ':': question.answer.text = question.answer.text[1:] questions.append(question) context['speeches'] = questions return context
def search(request): """ Returns messages corresponding to a query """ mlist_fqdn = request.GET.get("mlist") if mlist_fqdn is None: mlist = None else: try: mlist = MailingList.objects.get(name=mlist_fqdn) except MailingList.DoesNotExist: raise Http404("No archived mailing-list by that name.") if not is_mlist_authorized(request, mlist): return render(request, "hyperkitty/errors/private.html", { "mlist": mlist, }, status=403) query = '' results = EmptySearchQuerySet() sqs = RelatedSearchQuerySet() # Remove private non-subscribed lists if mlist is not None: sqs = sqs.filter(mailinglist__exact=mlist.name) else: excluded_mlists = MailingList.objects.filter( archive_policy=ArchivePolicy.private.value) if request.user.is_authenticated(): subscriptions = request.user.hyperkitty_profile.get_subscriptions() excluded_mlists = excluded_mlists.exclude( name__in=subscriptions.keys()) excluded_mlists = excluded_mlists.values_list("name", flat=True) sqs = sqs.exclude(mailinglist__in=excluded_mlists) # Sorting sort_mode = request.GET.get('sort') if sort_mode == "date-asc": sqs = sqs.order_by("date") elif sort_mode == "date-desc": sqs = sqs.order_by("-date") # Handle data if request.GET.get('q'): form = SearchForm( request.GET, searchqueryset=sqs, load_all=True) if form.is_valid(): query = form.cleaned_data['q'] results = form.search() else: form = SearchForm(searchqueryset=sqs, load_all=True) messages = paginate(results, page_num=request.GET.get('page')) for message in messages: if request.user.is_authenticated(): message.object.myvote = message.object.votes.filter(user=request.user).first() else: message.object.myvote = None context = { 'mlist' : mlist, 'form': form, 'messages': messages, 'query': query, 'sort_mode': sort_mode, 'suggestion': None, } if results.query.backend.include_spelling: context['suggestion'] = form.get_suggestion() return render(request, "hyperkitty/search_results.html", context)