def _haystack_search(self, content, queryset, max_results, partial, using):
from haystack.query import RelatedSearchQuerySet
from haystack.inputs import AutoQuery
if content.strip() == '':
return RelatedSearchQuerySet().none()
# Limit to the model bound to this manager, e.g. DataConcept.
# `load_all` ensures a single database hit when loading the objects
# that match
sqs = RelatedSearchQuerySet().models(self.model).load_all()
# If a non-default backend is being used, set which backend is to
# be used.
if using is not None:
sqs = sqs.using(using)
if partial:
# Autocomplete only works with N-gram fields.
sqs = sqs.autocomplete(text_auto=content)
else:
# Automatically handles advanced search syntax for negations
# and quoted strings.
sqs = sqs.filter(text=AutoQuery(content))
if queryset is not None:
sqs = sqs.load_all_queryset(self.model, queryset)
if max_results:
return sqs[:max_results]
return sqs
def search_products(
request,
template_name='satchless/search/haystack_predictive/products.html'):
form = forms.ProductPredictiveSearchForm(data=request.GET or None)
if form.is_valid():
results = form.search()
query = form.cleaned_data['q']
else:
results = RelatedSearchQuerySet().all()
results = results.load_all()
results = results.models(Product)
query = ''
products_per_page = getattr(settings,
'HAYSTACK_PREDICTIVE_PRODUCTS_PER_PAGE',
PRODUCTS_PER_PAGE)
paginator = Paginator(results, products_per_page)
try:
page = paginator.page(request.GET.get('page', 1))
except InvalidPage:
raise Http404()
return direct_to_template(request, template_name, {
'form': form,
'page': page,
'paginator': paginator,
'query': query,
})
def setUp(self):
super(LiveSolrSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SolrMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
print 'Reloading data...'
lssqstc_all_loaded = True
# Wipe it clean.
clear_solr_index()
# Force indexing of the content.
self.smmi.update()
def get_queryset(self):
query = self.request.GET.get("query", "")
order_by = self.request.GET.get("order", "")
try:
queryset = (
RelatedSearchQuerySet().models(ForumMessage).autocomplete(
auto=html.escape(query)))
except TypeError:
return []
if order_by == "date":
queryset = queryset.order_by("-date")
queryset = queryset.load_all()
queryset = queryset.load_all_queryset(
ForumMessage,
ForumMessage.objects.all().prefetch_related(
"topic__forum__edit_groups").prefetch_related(
"topic__forum__view_groups").prefetch_related(
"topic__forum__owner_club"),
)
# Filter unauthorized responses
resp = []
count = 0
max_count = 30
for r in queryset:
if count >= max_count:
return resp
if can_view(r.object, self.request.user) and can_view(
r.object.topic, self.request.user):
resp.append(r.object)
count += 1
return resp
def setUp(self):
super(LiveSolrSearchQuerySetTestCase, self).setUp()
# With the models registered, you get the proper bits.
import haystack
from haystack.sites import SearchSite
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_site = haystack.site
test_site = SearchSite()
test_site.register(MockModel, SolrMockModelSearchIndex)
haystack.site = test_site
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
print 'Reloading data...'
lssqstc_all_loaded = True
# Wipe it clean.
clear_solr_index()
# Force indexing of the content.
mockmodel_index = test_site.get_index(MockModel)
mockmodel_index.update()
def __init__(self, GET, *args, **kwargs):
# Save all GET parameters in case they match facets and not form fields
self.filters = GET
super(FeedSearchForm, self).__init__(GET, *args, **kwargs)
self.searcher = Searcher(model=Trial,
facets=my_facet_config,
queryset=RelatedSearchQuerySet())
def search(self, query):
if not query:
return EmptySearchQuerySet()
# OPTIMIZE: the number of public projects can increase substantially
# causing a really high number of project_ids to be sended to
# elasticsearch
projects = Project.objects.public_or_collaborate(self.request.user)
return RelatedSearchQuerySet()\
.filter(project_id__in=projects.values_list('pk', flat=True))\
.filter(SQ(content__contains=Clean(query)) |
SQ(title__contains=Clean(query)))
def get_results(self, request):
if not SEARCH_VAR in request.GET:
return super(AllChangeList, self).get_results(request)
# Note that pagination is 0-based, not 1-based.
sqs = RelatedSearchQuerySet().models(self.model).auto_query(
request.GET[SEARCH_VAR]).load_all()
if not request.user.is_superuser:
result = self.model.objects.distinct().filter(
Q(owners=request.user)
| Q(editor_groups__name__in=request.user.groups.values_list(
'name', flat=True)))
else:
result = self.model.objects.distinct()
if 'storage_object__publication_status__exact' in request.GET:
result = result.filter(
storage_object__publication_status__exact=request.
GET["storage_object__publication_status__exact"])
sqs = sqs.load_all_queryset(self.model, result)
paginator = Paginator(sqs, self.list_per_page)
# Fixed bug for pagination, count full_result_count
full_result_count = 0
for sq in sqs:
full_result_count += 1
result_count = paginator.count
# Get the number of objects, with admin filters applied.
can_show_all = result_count <= list_max_show_all(self)
multi_page = result_count > self.list_per_page
# Get the list of objects to display on this page.
try:
result_list = paginator.page(self.page_num + 1).object_list
# Grab just the Django models, since that's what everything else is
# expecting.
result_list = [result.object for result in result_list]
except InvalidPage:
result_list = ()
self.result_count = result_count
self.full_result_count = full_result_count
self.result_list = result_list
self.can_show_all = can_show_all
self.multi_page = multi_page
self.paginator = paginator
def create_queryset(user):
# ideally we'd do it this way, and not mention specific models here
# (instead doing visibility checking in their respective search_index.py files),
# but there's no way to pass a user into serach_index.load_all_queryset. yet.
#
# return RelatedSearchQuerySet()
qs = RelatedSearchQuerySet().load_all_queryset(
GroupTopic, GroupTopic.objects.visible(user))
qs = qs.load_all_queryset(Whiteboard, Whiteboard.objects.visible(user))
qs = qs.load_all_queryset(Event, Event.objects.visible(user))
# TODO: centralize this somewhere?
execlist = get_object_or_none(Community, slug='exec')
if execlist and execlist.user_is_member(user, True):
qs = qs.load_all_queryset(Activity, Activity.objects.all())
else:
qs = qs.load_all_queryset(Activity, Activity.objects.none())
return qs
def filter_queryset(self, request, queryset, view):
current_user = request.user
permissioned_qs = None
if current_user.is_staff:
permissioned_qs = queryset
elif not current_user.is_anonymous():
user_profile = UserProfile.objects.get(user=current_user)
permissioned_qs = queryset.filter(
Q(parent_id=user_profile.id,
parent_type=ContentType.objects.get_for_model(UserProfile))
| Q(parent_id__in=user_profile.organizations,
parent_type=ContentType.objects.get_for_model(
Organization)) | ~Q(public_access=ACCESS_TYPE_NONE))
else:
permissioned_qs = queryset.filter(~Q(
public_access=ACCESS_TYPE_NONE))
return super(ConceptContainerPermissionedSearchFilter,
self)._filter_queryset(request, permissioned_qs, view,
RelatedSearchQuerySet())
def build_form(self, *args, **kwargs):
self.searchqueryset = RelatedSearchQuerySet()
if self.request.GET.get('sort') == 'newest':
self.searchqueryset = self.searchqueryset.order_by('-start_date')
return super(CustomSearchView, self).build_form(*args, **kwargs)
def __init__(self, *args, **kwargs):
if not kwargs.get('searchqueryset', None):
kwargs['searchqueryset'] = RelatedSearchQuerySet()
super(ProductPredictiveSearchForm, self).__init__(*args, **kwargs)
def __init__(self, *args, **kwargs):
if kwargs.get('searchqueryset') is None:
kwargs['searchqueryset'] = RelatedSearchQuerySet()
super(SpeechForm, self).__init__(*args, **kwargs)
def get_query(self, search_term):
return RelatedSearchQuerySet().filter(content=search_term)
def filter_queryset(self, request, queryset, view):
return self._filter_queryset(request, queryset, view,
RelatedSearchQuerySet())
def search(request):
""" Returns messages corresponding to a query """
mlist_fqdn = request.GET.get("mlist")
if mlist_fqdn is None:
mlist = None
else:
try:
mlist = MailingList.objects.get(name=mlist_fqdn)
except MailingList.DoesNotExist:
raise Http404("No archived mailing-list by that name.")
if not is_mlist_authorized(request, mlist):
return render(request,
"hyperkitty/errors/private.html", {
"mlist": mlist,
},
status=403)
query = ''
results = EmptySearchQuerySet()
sqs = RelatedSearchQuerySet()
# Remove private non-subscribed lists
if mlist is not None:
sqs = sqs.filter(mailinglist__exact=mlist.name)
else:
excluded_mlists = MailingList.objects.filter(
archive_policy=ArchivePolicy.private.value)
if request.user.is_authenticated:
subscriptions = get_subscriptions(request.user)
excluded_mlists = excluded_mlists.exclude(
list_id__in=subscriptions.keys())
excluded_mlists = excluded_mlists.values_list("name", flat=True)
sqs = sqs.exclude(mailinglist__in=excluded_mlists)
# Sorting
sort_mode = request.GET.get('sort')
if sort_mode == "date-asc":
sqs = sqs.order_by("date")
elif sort_mode == "date-desc":
sqs = sqs.order_by("-date")
# Handle data
if request.GET.get('q'):
form = SearchForm(request.GET, searchqueryset=sqs, load_all=True)
if form.is_valid():
query = form.cleaned_data['q']
results = form.search()
else:
form = SearchForm(searchqueryset=sqs, load_all=True)
try:
emails = paginate(
results,
request.GET.get('page'),
request.GET.get('count'),
)
except Exception as e:
backend = settings.HAYSTACK_CONNECTIONS[DEFAULT_ALIAS]["ENGINE"]
if backend == "haystack.backends.whoosh_backend.WhooshEngine":
from whoosh.qparser.common import QueryParserError
search_exception = QueryParserError
if backend == "xapian_backend.XapianEngine":
from xapian import QueryParserError
search_exception = QueryParserError
if not isinstance(e, search_exception):
raise
emails = paginate([])
form.add_error(
"q",
ValidationError(
_('Parsing error: %(error)s'),
params={"error": e},
code="parse",
))
for email in emails:
if request.user.is_authenticated:
email.object.myvote = email.object.votes.filter(
user=request.user).first()
else:
email.object.myvote = None
context = {
'mlist': mlist,
'form': form,
'emails': emails,
'query': query,
'sort_mode': sort_mode,
'suggestion': None,
}
if results.query.backend.include_spelling:
context['suggestion'] = form.get_suggestion()
return render(request, "hyperkitty/search_results.html", context)
handler500 = "pinax.views.server_error"
urlpatterns = patterns("",
url(r"^", include("pins.urls")),
url(r"^admin/", include(admin.site.urls)),
url(r'^admin_tools/', include('admin_tools.urls')),
url(r"^about/", include("about.urls")),
url(r"^account/", include("account.urls")),
url(r"^profile/", include("profiles.urls")),
url(r'^comments/', include('django.contrib.comments.urls')),
url('^activity/', include('actstream.urls')),
url(r'^abuse/', include('abuse_reports.urls')),
url(r'^invite/', include('invite_friends.urls')),
url(r'^auth/', include('social_auth.urls')),
url(r'^cms/', include('cms.urls')),
)
urlpatterns = urlpatterns + patterns('haystack.views',
url(r'^search/$', SearchView(searchqueryset=RelatedSearchQuerySet()), name='haystack_search'),
)
if settings.SERVE_MEDIA:
urlpatterns += patterns("",
url(r"", include("staticfiles.urls"),),
url(r'^site_media/media/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.MEDIA_ROOT}),
url(r'^site_media/static/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.STATIC_ROOT}),
)
"events_recording.event_id = events_event. ID AND "
"events_recording.media_file_id = multimedia_mediafile. ID AND "
" events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'"
" GROUP BY events_event.id",
'audio_count':
"SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE "
"events_recording.event_id = events_event. ID AND "
"events_recording.media_file_id = multimedia_mediafile. ID AND "
" events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='audio'"
" GROUP BY events_event.id",
}))
return sqs
event_search = EventSearchView(form_class=EventSearchForm,
searchqueryset=RelatedSearchQuerySet())
def annotate_events(events):
return events.annotate(product_count=Count('products')).extra(
select={
'video_count':
"SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE "
"events_recording.event_id = events_event. ID AND "
"events_recording.media_file_id = multimedia_mediafile. ID AND "
" events_recording. STATE = 'Published' AND multimedia_mediafile.media_type='video'"
" GROUP BY events_event.id",
'audio_count':
"SELECT COUNT(*) FROM events_recording, multimedia_mediafile WHERE "
"events_recording.event_id = events_event. ID AND "
"events_recording.media_file_id = multimedia_mediafile. ID AND "
def plus_search(tags,
search,
search_types,
order=None,
in_group=None,
extra_filter=None):
items = get_resources_for_tag_intersection(tags)
q = None
for typ, info in search_types:
if info[0]:
typ_items = Q(**info[0])
if info[1]:
typ_items = typ_items & ~Q(**info[1])
elif info[1]:
typ_items = ~Q(**info[1])
if not q:
q = typ_items
else:
q = q | typ_items
if extra_filter:
q = q & Q(**extra_filter)
if q:
items = items.filter(q)
included = None
for obj_class, included_filter in object_type_filters.items():
objs = obj_class.objects.filter(**included_filter)
included_search = {
'content_type__model': obj_class.__name__.lower(),
'object_id__in': objs
}
if not included:
included = Q(**included_search)
included = included | Q(**included_search)
items = items.filter(included)
if in_group:
# this should be implemented using the code just above and an external search filter arg
page_ids = WikiPage.objects.filter(in_agent=in_group, stub=False)
wikipages = Q(**{
'content_type__model': 'wikipage',
'object_id__in': page_ids
})
resource_ids = Resource.objects.filter(in_agent=in_group, stub=False)
resources = Q(**{
'content_type__model': 'resource',
'object_id__in': resource_ids
})
q = resources | wikipages
items = items.filter(q)
results_map = {}
tag_intersection = []
if search:
results = RelatedSearchQuerySet().auto_query(search)
results_map = {}
if results:
all_results = results.load_all_queryset(GenericReference, items)
if order == 'relevance':
all_results = all_results.load_all(
) # this bit is quite evil and makes things really inefficient for large searches
# a better approach would be to get all the ids directly from the fulltext index and use them as a filter for GenericReferences
results_map['All'] = [item.object for item in all_results
] #we really really shouldn't do this
items_len = len(results_map['All'])
else:
search_results = [result.pk for result in all_results]
results_map['All'] = items.filter(
id__in=search_results).order_by(order)
items_len = results_map['All'].count()
else:
results_map = {'All': EmptySearchQuerySet()}
items_len = 0
else:
if items:
items = items.order_by('creator')
items_len = items.count()
results_map['All'] = items
else:
items_len = 0
results_map = {'All': EmptySearchQuerySet()}
if order == 'modified':
results_map['All'] = results_map['All'].order_by('-' + order)
elif order == 'display_name':
results_map['All'] = results_map['All'].order_by(order)
if 'All' in results_map:
tag_intersection = get_intersecting_tags(results_map['All'], n=15)
if len(search_types) > 1:
for typ, info in search_types:
if info[0]:
typ_items = items.filter(**info[0])
if info[1]:
typ_items = items.exclude(**info[1])
if search and results and order == 'relevance':
# why do this again when we could just separate results using python
typ_items = all_results.load_all_queryset(
GenericReference, typ_items)
typ_items = [item.object for item in typ_items
] #we really really shouldn't do this
if typ_items:
results_map[typ] = typ_items
else:
results_map = {'All': EmptySearchQuerySet()}
search_type_data = []
for typ, data in search_types:
if results_map.has_key(typ):
try:
type_len = results_map[typ].count()
except TypeError:
type_len = len(results_map[typ])
search_type_data.append((typ, data[2], results_map[typ], type_len))
return {
'All': results_map['All'],
'items_len': items_len,
'search_types': search_type_data,
'tag_intersection': tag_intersection
}
def test_empty(self):
self.assertEqual(len(DataConcept.objects.published()), 12)
self.assertEqual(RelatedSearchQuerySet().models(DataConcept).count(),
12)
self.assertEqual(len(DataConcept.objects.search('')), 0)
def get_context_data(self, **kwargs):
context = super(SAQuestionIndex, self).get_context_data(**kwargs)
context['ministers'] = []
ministers = Section.objects.filter(parent__heading='Questions')
ministers = sorted(ministers, key=questions_section_sort_key)
for minister in ministers:
context['ministers'].append({
'title':
minister.title.replace('Questions asked to the ', ''),
'slug':
minister.slug
})
context['orderby'] = 'recentanswers'
context['minister'] = 'all'
context['q'] = ''
#set filter values
for key in ('orderby', 'minister', 'q'):
if key in self.request.GET:
context[key] = self.request.GET[key]
if not context['orderby'] in ['recentquestions', 'recentanswers']:
context['orderby'] = 'recentquestions'
search_result_sections = []
if context['q'] != '':
#using a RelatedSearchQuerySet seems to result in fewer
#queries, although the same results can be achieved with a
#SearchQuerySet
query = RelatedSearchQuerySet().models(Speech)
query = query.filter(
tags__name__in=['question', 'answer'],
content=AutoQuery(context['q']),
).load_all()
all_speeches = Speech.objects.all().filter(
tags__name__in=['question', 'answer'])
if context['minister'] != 'all':
all_speeches = all_speeches.filter(
section__parent__slug=context['minister'])
query = query.load_all_queryset(Speech, all_speeches)
for result in query:
search_result_sections.append(result.object.section.id)
sections = Section \
.objects \
.filter(
parent__parent__heading='Questions'
) \
.select_related('parent') \
.prefetch_related('speech_set') \
.annotate(
earliest_date=Min('speech__start_date'),
smallest_id=Min('speech__id'),
number_speeches=Count('speech'),
latest_date=Max('speech__start_date'),
)
if context['minister'] != 'all':
sections = sections.filter(parent__slug=context['minister'])
if len(search_result_sections) > 0:
sections = sections.filter(id__in=search_result_sections)
if context['orderby'] == 'recentanswers':
sections = sections.filter(number_speeches__gt=1).order_by(
'-latest_date', '-smallest_id')
else:
sections = sections.order_by('-earliest_date', '-smallest_id')
paginator = Paginator(sections, 10)
page = self.request.GET.get('page')
try:
sections = paginator.page(page)
except PageNotAnInteger:
sections = paginator.page(1)
except EmptyPage:
sections = paginator.page(paginator.num_pages)
context['paginator'] = sections
#format questions and answers for the template
questions = []
for section in sections:
question = section.speech_set.all()[0]
question.questionto = section.parent.heading.replace(
'Questions asked to the ', '')
question.questionto_slug = section.parent.slug
#assume if there is more than one speech that the question is answered
if len(section.speech_set.all()) > 1:
question.answer = section \
.speech_set \
.all()[len(section.speech_set.all()) - 1]
#extract the actual reply from the reply text (replies
#often include the original question and other text,
#such as a letterhead)
text = question.answer.text.split('REPLY')
if len(text) > 1:
question.answer.text = text[len(text) - 1]
if question.answer.text[0] == ':':
question.answer.text = question.answer.text[1:]
questions.append(question)
context['speeches'] = questions
return context
def search(request):
""" Returns messages corresponding to a query """
mlist_fqdn = request.GET.get("mlist")
if mlist_fqdn is None:
mlist = None
else:
try:
mlist = MailingList.objects.get(name=mlist_fqdn)
except MailingList.DoesNotExist:
raise Http404("No archived mailing-list by that name.")
if not is_mlist_authorized(request, mlist):
return render(request, "hyperkitty/errors/private.html", {
"mlist": mlist,
}, status=403)
query = ''
results = EmptySearchQuerySet()
sqs = RelatedSearchQuerySet()
# Remove private non-subscribed lists
if mlist is not None:
sqs = sqs.filter(mailinglist__exact=mlist.name)
else:
excluded_mlists = MailingList.objects.filter(
archive_policy=ArchivePolicy.private.value)
if request.user.is_authenticated():
subscriptions = request.user.hyperkitty_profile.get_subscriptions()
excluded_mlists = excluded_mlists.exclude(
name__in=subscriptions.keys())
excluded_mlists = excluded_mlists.values_list("name", flat=True)
sqs = sqs.exclude(mailinglist__in=excluded_mlists)
# Sorting
sort_mode = request.GET.get('sort')
if sort_mode == "date-asc":
sqs = sqs.order_by("date")
elif sort_mode == "date-desc":
sqs = sqs.order_by("-date")
# Handle data
if request.GET.get('q'):
form = SearchForm(
request.GET, searchqueryset=sqs, load_all=True)
if form.is_valid():
query = form.cleaned_data['q']
results = form.search()
else:
form = SearchForm(searchqueryset=sqs, load_all=True)
messages = paginate(results, page_num=request.GET.get('page'))
for message in messages:
if request.user.is_authenticated():
message.object.myvote = message.object.votes.filter(user=request.user).first()
else:
message.object.myvote = None
context = {
'mlist' : mlist,
'form': form,
'messages': messages,
'query': query,
'sort_mode': sort_mode,
'suggestion': None,
}
if results.query.backend.include_spelling:
context['suggestion'] = form.get_suggestion()
return render(request, "hyperkitty/search_results.html", context)
