def test_signal_setup_handling(self):
old_ui = connections['default'].get_unified_index()
connections['default']._index = UnifiedIndex()
self.assertEqual(connections['default'].get_unified_index()._indexes_setup, False)
foo_1 = Foo.objects.create(
title='chekin sigalz',
body='stuff'
)
fi = connections['default'].get_unified_index().get_index(Foo)
fi.clear()
fi.update()
sqs = SearchQuerySet()
existing_foo = sqs.filter(id='discovery.foo.1')[0]
self.assertEqual(existing_foo.text, u'stuff')
fi.clear()
foo_1 = Foo.objects.get(pk=1)
foo_1.title = 'Checking signals'
foo_1.body = 'Stuff.'
# This save should trigger an update.
foo_1.save()
self.assertEqual(connections['default'].get_unified_index()._indexes_setup, True)
sqs = SearchQuerySet()
new_foo = sqs.filter(id='discovery.foo.1')[0]
self.assertEqual(new_foo.text, u'Stuff.')
connections['default']._index = old_ui
def search(self):
# First, store the SearchQuerySet received from other processing.
sqs = super(CustomSearchForm, self).search()
# import ipdb; ipdb.set_trace();
if not self.is_valid():
return self.no_query_found()
# Check to see if a start_date was chosen.
if self.cleaned_data['q']:
sqs = SearchQuerySet().facet('author')
sqs_content = sqs.filter(content_auto=self.cleaned_data['q'])
sqs_title = sqs.filter(title_auto=self.cleaned_data['q'])
sqs = sqs_content|sqs_title
if self.cleaned_data['start_date']:
sqs = sqs.filter(pub_date__gte=self.cleaned_data['start_date'])
# Check to see if an end_date was chosen.
if self.cleaned_data['end_date']:
sqs = sqs.filter(pub_date__lte=self.cleaned_data['end_date'])
if self.selected_facets:
faceted_query_set = EmptySearchQuerySet()
for facet in self.selected_facets:
if 'author' in facet:
author_exact = facet.split(':')
author_name = author_exact[len(author_exact)-1]
faceted_query_set = SearchQuerySet().filter(author=author_name)|faceted_query_set
sqs = sqs&faceted_query_set
return sqs
def search_filter(self, filters=None, *args, **kwargs):
sqs = SearchQuerySet()
user = kwargs.get('user', None)
groups = []
if user and user.is_authenticated():
groups = [g.pk for g in user.group_set.all()]
admin = user.profile.is_superuser
# permission filters
if user:
if not user.profile.is_superuser:
if not user.is_anonymous():
# (status+status_detail+(anon OR user)) OR (who_can_view__exact)
anon_query = Q(allow_anonymous_view=True)
user_query = Q(allow_user_view=True)
sec1_query = Q(status=True, status_detail='active')
user_perm_q = Q(users_can_view__in=[user.pk])
group_perm_q = Q(groups_can_view__in=groups)
query = reduce(operator.or_, [anon_query, user_query])
query = reduce(operator.and_, [sec1_query, query])
query = reduce(operator.or_, [query, user_perm_q, group_perm_q])
else:
sqs = sqs.filter(allow_anonymous_view=True)
else:
sqs = sqs.filter(allow_anonymous_view=True)
# custom filters
for filter in filters:
sqs = sqs.filter(content='"%s"' % filter)
return sqs.models(self.model)
def search(request):
request.session['prev_url'] = request.build_absolute_uri()
q = request.GET.get('q')
levels = request.GET.get('levels')
counties = request.GET.get('counties')
categories = request.GET.get('categories')
# Text query
if q is None:
results = SearchQuerySet().all()
else:
results = SearchQuerySet().filter(content=AutoQuery(q))
# Add filters
results = results.filter(state=request.state_filter)
if levels:
levels = levels.split('-')
results = results.filter(level__in=levels)
if categories:
categories = categories.split('-')
results = results.filter(category__in=categories)
if counties:
counties = counties.split('-')
results = results.filter(county__in=counties)
rtitle = "Showing %s results" % len(results)
levels = ['state', 'county', 'municipal']
return render(request, 'bids/search.html',
{'result_title': rtitle, 'results': results, 'cats': BidCategory.choices(),
'counties': County.choices(), 'levels': levels })
def get_queryset(self):
params = self.request.GET
highlighted = params.get('highlighted') == 'true'
if highlighted:
return Project.objects.filter(highlighted=highlighted)
query = params.get('query', None)
cause = params.get('cause', None)
skill = params.get('skill', None)
city = params.get('city', None)
nonprofit = params.get('nonprofit', None)
if nonprofit:
nonprofit = Nonprofit.objects.get(user__slug=nonprofit)
queryset = Project.objects.filter(nonprofit=nonprofit)
else:
queryset = SearchQuerySet().models(Project)
queryset = queryset.filter(causes=cause) if cause else queryset
queryset = queryset.filter(skills=skill) if skill else queryset
queryset = queryset.filter(city=city) if city else queryset
queryset = queryset.filter(content=query).boost(query, 2) if query else queryset
results = [ r.pk for r in queryset]
return Project.objects.filter(pk__in=results, deleted=False, closed=False, published=True).order_by('?')
def search(self, query=None, *args, **kwargs):
"""
haystack to query corporate memberships.
Returns a SearchQuerySet
"""
from corporate_memberships.models import CorporateMembership
from perms.utils import is_admin
user = kwargs.get('user', None)
if user.is_anonymous():
return SearchQuerySet().models().none()
is_an_admin = is_admin(user)
sqs = SearchQuerySet().models(CorporateMembership)
if query:
sqs = sqs.filter(content=sqs.query.clean(query))
else:
sqs = sqs.all()
if not is_an_admin:
# reps__contain
sqs = sqs.filter(Q(content='rep\:%s' % user.username) |
Q(creator=user) |
Q(owner=user)).filter(status_detail='active')
return sqs
def __init__(self, *args, user, report_ids=(), **kwargs):
sqs = SearchQuerySet().models(Report)
# Ensure anonymous/public users cannot see non-public reports in all
# cases.
if not user.is_active:
if report_ids:
sqs = sqs.filter(SQ(id__in=report_ids) | SQ(is_public=True))
else:
sqs = sqs.filter(is_public=True)
super().__init__(*args, searchqueryset=sqs, **kwargs)
self.user = user
self.report_ids = report_ids
# Only certain fields on this form can be used by members of the
# public.
if not user.is_active:
for name in self.fields.keys():
if name not in self.public_fields:
self.fields.pop(name)
if user.is_anonymous():
if report_ids:
source_field = self.fields['source']
source_choices = source_field.choices
source_field.choices = [
(value, label) for (value, label) in source_choices if value != 'invited']
else:
self.fields.pop('source')
def search(self):
sqs = SearchQuerySet().load_all()
if not self.is_valid():
return sqs
if self.cleaned_data['q']:
#search input of a code, contains at least '_YYYY_'
if not re.match('([a-zA-Z]*_?[EI])?_[0-9]{4}_([0-9]{3}_[0-9]{3})?', self.cleaned_data.get('q')):
sqs = sqs.filter(content__contains=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type')
else:
sqs = sqs.filter(code__exact=self.cleaned_data['q']).facet('item_acces').facet('item_status').facet('digitized').facet('recording_context').facet('physical_format').facet('media_type')
print(self.cleaned_data['q'])
for facet in self.selected_facets:
if ":" not in facet:
continue
field, value = facet.split(":", 1)
if value:
if value == 'viewable':
sqs = sqs.narrow('item_acces:full OR item_acces:mixed').narrow('digitized:T')
else:
sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value)))
return sqs
def search(request, template_name="django_messages/search.html"):
search_term = request.GET.get("qs")
search_filter = request.GET.get("search")
results = SearchQuerySet().filter(participants=request.user.pk).filter(
SQ(content__icontains=search_term) | SQ(participant_last_names__istartswith=search_term)
).order_by('-last_message')
if request.GET.get('search')==u'sent':
results = results.filter(participant_sent=u"{}-{}".format(request.user.pk, True))
elif request.GET.get('search')==u'inbox':
results = results.filter(participant_archived=u"{}-{}".format(request.user.pk, False))
elif request.GET.get('search')==u'archives':
results = results.filter(participant_archived=u"{}-{}".format(request.user.pk, True))
else:
pass
results = results
paginated_messages = Paginator(results, 10)
page = request.GET.get('page', 1)
try:
paginated_list = paginated_messages.page(page)
except EmptyPage:
paginated_list = paginated_messages.page(paginated_messages.num_pages)
except PageNotAnInteger:
paginated_list = paginated_messages.page(1)
return render_to_response(template_name, {
"thread_results": paginated_list,
"search_term": search_term,
"search_filter": search_filter,
'page_type': 'search',
}, context_instance=RequestContext(request))
def search(self):
if not hasattr(self, 'cleaned_data'):
return self.no_query_found()
elif not self.cleaned_data['q']:
sqs = SearchQuerySet()
else:
sqs = super(ToolForm, self).search()
bool_fields = [
'supports_osx', 'supports_windows', 'supports_linux', 'supports_web_based',
'capture', 'capture_os', 'capture_code', 'capture_workflow', 'capture_data',
'representation', 'representation_descriptive_only', 'representation_executable',
'replicability',
'modifiability',
# 'portability',
'longevity', 'longevity_archiving', 'longevity_upgrading',
'document_linkage', 'document_linkage_by_reference', 'document_linkage_inline',
'experiment_sharing', 'experiment_sharing_archival', 'experiment_sharing_hosted_execution'
]
for field in bool_fields:
if field in self.cleaned_data and self.cleaned_data[field]:
sqs = sqs.filter(**{field:self.cleaned_data[field]=='True'})
if 'portability' in self.cleaned_data and self.cleaned_data['portability']:
sqs = sqs.filter(portability=self.cleaned_data['portability'])
return sqs
def get_section_data(self, section):
defaults = self.search_sections[section]
extra_filter = defaults.get('filter', {})
filter_args = extra_filter.get('args', [])
filter_kwargs = extra_filter.get('kwargs', {})
extra_exclude = defaults.get('exclude', {})
query = SearchQuerySet().models(defaults['model'])
if extra_exclude:
query = query.exclude(**extra_exclude)
query = query.filter(
content=AutoQuery(self.query),
*filter_args,
**filter_kwargs
)
if self.start_date_range:
query = query.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
query = query.filter(start_date__lte=self.end_date_range)
if self.order == 'date':
query = query.order_by('-start_date')
result = defaults.copy()
result['results'] = query.highlight()
result['results_count'] = result['results'].count()
result['section'] = section
result['section_dashes'] = section.replace('_', '-')
return result
def get(self, request, *args, **kwargs):
term = request.GET.get('term', '').strip()
if len(term) > 0:
sqs = SearchQuerySet().using(settings.FIAS_HAYSTACK_CONNECTION_ALIAS).models(AddrObj)
aolevels = request.GET.get('aolevels', '')
aolevels = list(map(int, filter(lambda s: s not in EMPTY_VALUES, map(strip, aolevels.split(',')))))
if aolevels:
sqs = sqs.filter(aolevel__in=aolevels)
socrs = request.GET.get('socrs', '')
socrs = list(filter(lambda s: s not in EMPTY_VALUES, map(strip, socrs.split(','))))
if socrs:
sqs = sqs.filter(shortname__in=socrs)
cleaned_words = [sqs.query.clean(word.strip()) for word in term.split(' ')]
query_bits = [SQ(text_auto=word) for word in cleaned_words if word]
if query_bits:
sqs = sqs.filter(six.moves.reduce(operator.__or__, query_bits))
items = sqs
items = items[:50]
else:
items = []
return JsonResponse({
'results': [
{'id': l.aoguid, 'text': l.full_name, 'level': l.aolevel} for l in items
],
'more': False
})
def cruce10(form):
#Revizar que este en el estado seleccionado
if form.cleaned_data['codigo'] != '':
sqs = SearchQuerySet()
sqs = sqs.filter(estado = form.cleaned_data['estado']).filter(codigo = form.cleaned_data['codigo'])
if sqs != []:
for yac in sqs:
if len(yac.manifestacion) != 0:
if yac.manifestacion[0].encode('utf8') != 'None':
if int(yac.manifestacion[0]) <= 22:
yac.manifestacion = MANIFESTACIONES[int(yac.manifestacion[0])]
return {
'yacimientos': sqs}
else:
return {
'yacimientos': sqs}
#Buscar en todo ese estado
elif form.cleaned_data['codigo'] == '':
sqs = SearchQuerySet()
sqs = sqs.filter(estado = form.cleaned_data['estado'])
for yac in sqs:
if len(yac.manifestacion) != 0:
if yac.manifestacion[0].encode('utf8') != 'None':
if int(yac.manifestacion[0]) <= 22:
yac.manifestacion = MANIFESTACIONES[int(yac.manifestacion[0])]
return {
'yacimientos': sqs}
def build_form(self, form_kwargs=None):
log = logging.getLogger(".".join((__name__, "PostSearch")))
request = self.request
log.debug("which author has domain %r?", request.META["HTTP_HOST"])
try:
self.author = User.objects.get(authorsite__site__domain=request.META["HTTP_HOST"])
except User.DoesNotExist:
log.debug(" no such author! no results at all!")
self.author = None
sqs = SearchQuerySet().none()
else:
sqs = SearchQuerySet().filter(author_pk=self.author.pk)
# What visibility of posts can the searcher see?
if request.user.is_anonymous():
log.debug(" viewer is anonymous, so only %s's public posts", self.author.username)
sqs = sqs.filter(private=0)
elif request.user.pk == self.author.pk:
log.debug(" viewer is %s, so all their posts", self.author.username)
else:
# TODO: honor trust groups instead of giving everyone else only public posts
log.debug(" viewer is logged in as somebody else, so only %s's public posts", self.author.username)
sqs = sqs.filter(private=0)
self.searchqueryset = sqs
return super(PostSearch, self).build_form(form_kwargs)
def get_global_context(self, context):
# Find all the models to search over...
models = set(self.search_sections[section]["model"] for section in self.search_sections)
show_top_hits = self.page == "1" or not self.page
top_hits_ids = []
if show_top_hits:
context["top_hits"] = []
for section, max_for_top_hits in SearchBaseView.top_hits_under.items():
data = self.get_section_data(section)
if data["results_count"] <= max_for_top_hits:
context["top_hits"] += data["results"]
top_hits_ids = set(r.id for r in context["top_hits"])
sqs = SearchQuerySet().models(*list(models))
# Exclude anything that will already have been shown in the top hits:
for top_hit_id in top_hits_ids:
sqs = sqs.exclude(id=top_hit_id)
sqs = sqs.exclude(hidden=True).filter(content=AutoQuery(self.query)).highlight()
if self.start_date_range:
sqs = sqs.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
sqs = sqs.filter(start_date__lte=self.end_date_range)
if self.order == "date":
sqs = sqs.order_by("-start_date")
context["paginator"] = Paginator(sqs, self.results_per_page)
context["page_obj"] = self.get_paginated_results(context["paginator"])
return context
def get_results(self, name, branch, year, offset, branch_facet, year_facet, city_facet):
if year_facet:
year_facet = [int(x) for x in year_facet.split(",")]
sqs = SearchQuerySet().facet('branch')
sqs = sqs.facet('year_of_passing')
sqs = sqs.facet('city')
if name:
sqs = sqs.auto_query(name)
if branch:
sqs = sqs.filter(branch_exact=branch)
if year:
sqs = sqs.filter(year_of_passing_exact=year)
if branch_facet:
sqs = sqs.filter(branch_exact=branch_facet)
if year_facet:
sqs = sqs.filter(year_of_passing_exact__in=year_facet)
if city_facet:
sqs = sqs.filter(city_exact=city_facet)
offsetvalue = int(offset)
results = sqs.order_by('name')[offsetvalue:offsetvalue + INITIAL_RESULTS_COUNT]
resultcount = len(results)
return results, resultcount
def get_queryset(self):
params = self.request.GET
highlighted = params.get("highlighted") == "true"
if highlighted:
return Project.objects.filter(highlighted=highlighted)
query = params.get("query", None)
cause = params.get("cause", None)
skill = params.get("skill", None)
city = params.get("city", None)
nonprofit = params.get("nonprofit", None)
if nonprofit:
nonprofit = Nonprofit.objects.get(user__slug=nonprofit)
queryset = Project.objects.filter(nonprofit=nonprofit)
else:
queryset = SearchQuerySet().models(Project)
queryset = queryset.filter(causes=cause) if cause else queryset
queryset = queryset.filter(skills=skill) if skill else queryset
queryset = queryset.filter(content=query) if query else queryset
queryset = queryset.filter(city=city) if city != u"" and int(city) else queryset
results = [q.pk for q in queryset]
if city == "0":
return Project.objects.filter(
pk__in=results, deleted=False, closed=False, published=True, work__can_be_done_remotely=True
).order_by("-highlighted")
else:
return Project.objects.filter(pk__in=results, deleted=False, closed=False, published=True).order_by(
"-highlighted"
)
def get_section_data(self, section):
defaults = self.search_sections[section]
extra_filter = defaults.get("filter", {})
filter_args = extra_filter.get("args", [])
filter_kwargs = extra_filter.get("kwargs", {})
extra_exclude = defaults.get("exclude", {})
query = SearchQuerySet().models(defaults["model"])
if extra_exclude:
query = query.exclude(**extra_exclude)
query = query.filter(content=AutoQuery(self.query), *filter_args, **filter_kwargs)
if self.start_date_range:
query = query.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
query = query.filter(start_date__lte=self.end_date_range)
if self.order == "date":
query = query.order_by("-start_date")
result = defaults.copy()
result["results"] = query.highlight()
result["results_count"] = result["results"].count()
result["section"] = section
result["section_dashes"] = section.replace("_", "-")
return result
def volunteer_page(request):
# Get the user comfort languages list
user_langs = get_user_languages_from_request(request)
relevant = SearchQuerySet().result_class(VideoSearchResult) \
.models(Video).filter(video_language_exact__in=user_langs) \
.filter_or(languages_exact__in=user_langs) \
.order_by('-requests_count')
featured_videos = relevant.filter(
featured__gt=datetime.datetime(datetime.MINYEAR, 1, 1)) \
.order_by('-featured')[:5]
popular_videos = relevant.order_by('-week_views')[:5]
latest_videos = relevant.order_by('-edited')[:15]
requested_videos = relevant.filter(requests_exact__in=user_langs)[:5]
context = {
'featured_videos': featured_videos,
'popular_videos': popular_videos,
'latest_videos': latest_videos,
'requested_videos': requested_videos,
'user_langs':user_langs,
}
return render_to_response('videos/volunteer.html', context,
context_instance=RequestContext(request))
def get_user_events_sqs(user):
events = SearchQuerySet().models(Event)
if is_student(user):
return events.filter(SQ(is_public=True) | SQ(invitees=user.id))
if is_campus_org(user):
return events.filter(owner=user.id)
return events.filter(SQ(owner=user.id) | SQ(attending_employers=user.recruiter.employer.id))
def _specific_view(user, obj):
"""
determines if a user has specific permissions to view the object.
note this is based only on:
(users_can_view contains user)
+
(groups_can_view contains one of user's groups)
"""
sqs = SearchQuerySet()
sqs = sqs.models(obj.__class__)
groups = [g.pk for g in user.group_set.all()]
q_primary_key = SQ(primary_key=obj.pk)
q_groups = SQ(groups_can_view__in=groups)
q_users = SQ(users_can_view__in=[user.pk])
if groups:
sqs = sqs.filter(q_primary_key & (q_groups | q_users))
else:
sqs = sqs.filter(q_primary_key & q_users)
if sqs:
# Make sure the index isn't doing something unexpected with the query,
# like when the Whoosh StopFilter caused the primary_key portion of the
# query to be ignored.
assert len(sqs) == 1, "Index returned an unexpected result set when searching for view permissions on an object"
return True
return False
class AutocompleteQueryTestCase(TestCase):
fixtures = ['mock_persons']
def setUp(self):
PersonIndex().reindex()
self.sqs = SearchQuerySet().all()
def test_filter_CharField(self):
# Make sure we only get exact hit when searching using
# the `CharField` fields
self.assertEqual(self.sqs.filter(firstname='abel').count(),
Person.objects.filter(firstname__iexact='abel').count())
self.assertEqual(self.sqs.filter(lastname='hood').count(),
Person.objects.filter(lastname__iexact='hood').count())
def test_filter_AutocompleteEdgeNgramField(self):
# Make sure we get results for all tokens indexed using
# the `AutocompleteEdgeNgramField`. We should get match on
# both firstname and lastname.
self.assertEqual(self.sqs.autocomplete(q='d').count(),
Person.objects.filter(Q(firstname__istartswith='d') |
Q(lastname__istartswith='d')).count())
self.assertEqual(self.sqs.autocomplete(q='ab').count(),
Person.objects.filter(Q(firstname__istartswith='ab') |
Q(lastname__istartswith='ab')).count())
def search(request):
"""Use haystack to find search results"""
term = request.GET.get('q', '')
version = request.GET.get('version', '')
regulation = request.GET.get('regulation', '')
try:
page = int(request.GET.get('page', '0'))
except ValueError:
page = 0
if not term:
return user_error('No query term')
query = SearchQuerySet().models(Regulation).filter(content=term)
if version:
query = query.filter(version=version)
if regulation:
query = query.filter(regulation=regulation)
start, end = page * PAGE_SIZE, (page+1) * PAGE_SIZE
return success({
'total_hits': len(query),
'results': transform_results(query[start:end])
})
def cruce19(form):
filters = ['litica',
'ceramica',
'oseo',
'concha',
'carbon no superficial',
'mitos',
'cementerios',
'monticulos',
'otros'
]
list = 0
try:
list = filters.index(form.cleaned_data['manifasociadas'].lower()) + 1
sqs = SearchQuerySet()
sqs = sqs.filter(pais = 'Venezuela').filter(manifasociadas = list)
yacimientos = {}
lista = []
for estado in ESTADOS:
lista = sqs.filter(estado = estado)
yacimientos[estado] = lista
return {
'manifa':form.cleaned_data['manifasociadas'],
'yacimientos': yacimientos}
except:
return {}
def get_content(self):
qs = SearchQuerySet()
search_fields = {
'q': 'text',
'controlling_bodies': 'controlling_body',
'statuses': 'status',
'file_types': 'file_type',
}
for key, val in self.filter.iteritems():
if key in search_fields:
field = search_fields[key]
else:
field = key
if val in ([], {}, '', (), None):
continue
if isinstance(val, list):
for item in val:
qs = qs.filter(**{field: item})
else:
qs = qs.filter(**{field: val})
return qs.order_by('order_date')
def search(self):
sqs = SearchQuerySet()
sqs = sqs.facet('bureaux').facet(
'section').facet('annee').facet('partenaire')
if self.q:
sqs = sqs.filter(content=sqs.query.clean(self.q))
if self.courant:
sqs = sqs.filter(date_fin__gte=datetime.date.today())
if self.cloture:
sqs = sqs.filter(date_fin__lt=datetime.date.today())
self.selected_facets = list(
set(self.selected_facets.split('&') + self.selected_facets_get))
for facet in self.selected_facets:
if "__" not in facet:
continue
field, value = facet.split("__", 1)
if value:
sqs = sqs.narrow(u'%s:"%s"' % (field, sqs.query.clean(value)))
return sqs
def _specific_view(user, obj):
"""
determines if a user has specific permissions to view the object.
note this is based only on:
(users_can_view contains user)
+
(groups_can_view contains one of user's groups)
"""
sqs = SearchQuerySet()
sqs = sqs.models(obj.__class__)
groups = [g.pk for g in user.group_set.all()]
q_primary_key = SQ(primary_key=obj.pk)
q_groups = SQ(groups_can_view__in=groups)
q_users = SQ(users_can_view__in=[user.pk])
if groups:
sqs = sqs.filter(q_primary_key & (q_groups | q_users))
else:
sqs = sqs.filter(q_primary_key & q_users)
if sqs:
return True
return False
def get_queryset(self):
if self.request.method in SAFE_METHODS:
# Listing/searching.
validate_request_on_app_id(self.request)
if "q" in self.request.query_params:
search_q = self.request.query_params["q"]
else:
# No query is provided. Full list is for authenticated requests only.
if not self.request.user.is_authenticated():
raise NotAuthenticated("")
return Picture.objects.order_by("item_id")
search_q = search_q.strip()
search_q = search_q.lower()
search_q = re.sub(r"\s+", " ", search_q)
search_q = search_q.replace("-", " ")
search_q = search_q.replace(".", "")
search_q = search_q.replace("'", "")
search_q = search_q.strip()
if not search_q:
return Picture.objects.none()
sqs = SearchQuerySet().models(Picture)
sqs = sqs.filter(content=AutoQuery(search_q))
pictures = [sr.object for sr in sqs.load_all()]
if not pictures:
# Auto-correction.
sugg = sqs.spelling_suggestion()
if (sugg is not None) and ("suggestion" in sugg) and sugg["suggestion"]:
sqs = SearchQuerySet().models(Picture)
sqs = sqs.filter(content=sugg["suggestion"][0])
pictures = [sr.object for sr in sqs.load_all()]
# Exclude hidden.
pictures = [picture for picture in pictures if not picture.is_hidden]
# Randomize leading good ones.
first_non_good_picture_pos = None
for i, picture in enumerate(pictures):
if not picture.is_good:
first_non_good_picture_pos = i
break
if first_non_good_picture_pos is not None:
leading_good_pictures = pictures[:first_non_good_picture_pos]
random.shuffle(leading_good_pictures)
pictures = leading_good_pictures + pictures[first_non_good_picture_pos:]
if BOOST_SEARCH_RESULTS_ON_NUM_USES:
pictures = boost_search_results_on_num_uses(pictures)
return pictures
else:
return Picture.objects.all()
def get_context_data(self, **kwargs):
sqs = SearchQuerySet().models(Boot).order_by('-star_count_month')
kwargs['trending'] = {
'projects': sqs.filter(type_exact=Boot.TYPE_PROJECT),
'apps': sqs.filter(type_exact=Boot.TYPE_APP),
'cookiecutters': sqs.filter(type_exact=Boot.TYPE_COOKIECUTTER),
}
return super(IndexView, self).get_context_data(**kwargs)
def get_queryset(self, query_args):
# Create new queryset
qs = SearchQuerySet()
# Are we searching all models or just a specific one (depends on
# parameter set in View instantiation)
if self.search_model is not None:
qs = qs.models(self.search_model)
# Do we have a query or are we just getting all of them?
if 'q' in query_args:
qry = query_args['q']
# fuzzify search
qry = u'{}~'.format(qry.replace(' ', '~ '))
qs = qs.auto_query(qry)
elif 'parl_id' in query_args:
qs = qs.filter(parl_id=query_args['parl_id'])
if 'llp_numeric' in query_args:
qs = qs.filter(llps_numeric=query_args['llp_numeric'])
return (qs.all(), None)
# Filter by facets
if query_args['facet_filters']:
for facet_field in query_args['facet_filters'].keys():
# We use narrow to limit the index entries beforehand, but
# need to use filter afterwards to remove partly correct results
# For instance, searching for Steyr (Oberoesterreich) yielded
# everyone from Oberoesterreich until filtering by it again.
qs = qs.narrow(u"{}:{}".format(
facet_field,
qs.query.clean(query_args['facet_filters'][facet_field]))
).filter(
**{
facet_field: query_args['facet_filters'][facet_field]}
)
# Retrieve facets and facet_counts
facet_counts = []
if self.facet_fields:
facets = qs
for facet_field in self.facet_fields:
if self.facet_fields[facet_field]['type'] == 'date':
facets = facets.date_facet(
facet_field,
start_date=datetime.date(1900, 1, 1),
end_date=datetime.date(2050, 1, 1),
gap_by='month')
if self.facet_fields[facet_field]['type'] == 'field':
facets = facets.facet(facet_field)
facet_counts = facets.facet_counts()
# Get results and return them
if 'only_facets' in query_args:
result = {}
else:
result = qs.all()
return (result, facet_counts)
def search(request):
result = {}
q = request.GET.get('q', '')
q = ' '.join(group_words(q))
page = request.GET.get('page', '')
size = request.GET.get('size', '')
code = request.GET.get('code', '')
dept_required = request.GET.get('dept_required', '')
sortby_param = request.GET.get('sort', '')
reverse_param = request.GET.get('reverse', '')
ys = request.GET.get('ys', SEMESTER)
page_size = size or 10
sortby = sortby_param or 'time_token'
reverse = True if reverse_param == 'true' else False
rev_sortby = '-' + sortby if reverse else sortby
courses = SearchQuerySet().filter()
if dept_required:
try:
courses = Department.objects.get(
ys=ys, dept_name=dept_required).required_course.all()
except:
pass
if courses:
result['type'] = 'required'
page_size = courses.count()
else:
courses = courses.filter(content=AutoQuery(q))
if code:
courses = courses.filter(code__contains=code)
if code in ['GE', 'GEC']:
core = request.GET.get(code.lower(), '')
if core:
courses = courses.filter(ge__contains=core)
courses = courses.order_by(rev_sortby)
paginator = Paginator(courses, page_size)
try:
courses_page = paginator.page(page)
except PageNotAnInteger:
# If page is not an integer, deliver first page.
courses_page = paginator.page(1)
except EmptyPage:
# If page is out of range (e.g. 9999), deliver last page of results.
courses_page = paginator.page(paginator.num_pages)
course_list = [
{k: v
for (k, v) in x.__dict__.items() if not k.startswith('_')} for x in
[x if dept_required else x.object for x in courses_page.object_list]
]
result['total'] = courses.count()
result['page'] = courses_page.number
result['courses'] = course_list
result['page_size'] = page_size
return JsonResponse(result)
def search(request, template='search/search.html', load_all=True,
form_class=ModelSearchForm, searchqueryset=None,
context_class=RequestContext, extra_context=None, results_per_page=None):
"""
A more traditional view that also demonstrate an alternative
way to use Haystack.
Useful as an example of for basing heavily custom views off of.
Also has the benefit of thread-safety, which the ``SearchView`` class may
not be.
Template:: ``search/search.html``
Context::
* form
An instance of the ``form_class``. (default: ``ModelSearchForm``)
* page
The current page of search results.
* paginator
A paginator instance for the results.
* query
The query received by the form.
"""
query = ''
data_domain_ids = []
results = SearchQuerySet()
geo_record, indicator = None, None
try:
geo_record = GeoRecord.objects.get(
id=request.GET.get('gid', None))
except (GeoRecord.DoesNotExist, ValueError):
# the navigation template tag would pick DEFAULT_GEO_RECORD_ID if no
# georecord is passed, but we should select it manually here so
# thumbnails can be rendered in the proper context
geo_record = GeoRecord.objects.get(id=settings.DEFAULT_GEO_RECORD_ID)
indicator_results = None
data_display_results = None
def _value_to_display(indicator_result):
datapoint = indicator_result.object.datapoint_set.filter(record=geo_record).order_by('-time__sort')
value = None
if datapoint:
value = datapoint[0].value_set.all()
if value:
value = value[0]
return (indicator_result, value)
if request.GET.get('q') and request.GET.get('q') != None:
data_domain_ids = map(lambda id: int(id), request.GET.getlist('data_domain'))
query = request.GET.get('q')
results = results.filter(content=query)
indicator_results = results.filter(records=geo_record.id).models(Indicator)
data_display_results = results.filter(record_id=geo_record.id).models(DataDisplay)
if data_domain_ids:
indicator_results = indicator_results.filter(domains__in=data_domain_ids)
data_display_results = data_display_results.filter(domains__in=data_domain_ids)
context = {
'indicator_results': map(_value_to_display, indicator_results),
'data_display_results': data_display_results,
'total_results': data_display_results.count() + indicator_results.count(),
'query': query,
'geo_record': geo_record,
'indicator': indicator,
'data_domains': DataDomain.objects.all(),
'data_domain_ids': data_domain_ids,
'beta':settings.BETA
}
else:
context = {}
return render_to_response(template, context, context_instance=context_class(request))
def get_context_data(self, **kwargs):
context = super(SearchView, self).get_context_data(**kwargs)
context['request'] = self.request
name = self.request.GET.get("name", '')
branch = self.request.GET.get("branch", '')
year = self.request.GET.get("year_of_passing", '')
branch_facet = self.request.GET.get("branch_facet", '')
year_facet = self.request.GET.get("year_of_passing_facet", '')
# results = self.get_results(name, branch, year, offset, branch_facet, year_facet)
if year_facet:
year_facet = [int(x) for x in year_facet.split(",")]
city_facet = self.request.GET.get("city_facet", "")
sqs = SearchQuerySet().facet('branch')
sqs = sqs.facet('year_of_passing')
sqs = sqs.facet('city')
if name or branch or year:
context['form'] = ProfileSearchBasicForm(self.request.GET)
if name:
sqs = sqs.auto_query(name)
context['name_selected'] = name
if branch:
sqs = sqs.filter(branch_exact=branch)
context['branch_selected'] = branch
if year:
sqs = sqs.filter(year_of_passing_exact=year)
context['year_selected'] = year
else:
context['form'] = ProfileSearchBasicForm()
context['facets'] = sqs.facet_counts()
# Horrible hardcoding - need to tweak it - By Srihari
# To compute the facet counts
if branch_facet:
temp = sqs.filter(branch_exact=branch_facet)
context['facets']['fields']['year_of_passing'] = temp.filter(
city_exact=city_facet).facet_counts()['fields'][
'year_of_passing'] if city_facet else temp.facet_counts(
)['fields']['year_of_passing']
elif city_facet:
context['facets']['fields']['year_of_passing'] = sqs.filter(
city_exact=city_facet).facet_counts(
)['fields']['year_of_passing']
if year_facet:
temp = sqs.filter(year_of_passing_exact__in=year_facet)
context['facets']['fields']['branch'] = temp.filter(
city_exact=city_facet).facet_counts(
)['fields']['branch'] if city_facet else temp.facet_counts(
)['fields']['branch']
elif city_facet:
context['facets']['fields']['branch'] = sqs.filter(
city_exact=city_facet).facet_counts()['fields']['branch']
if year_facet:
temp = sqs.filter(year_of_passing_exact__in=year_facet)
context['facets']['fields']['city'] = temp.filter(
branch_exact=branch_facet).facet_counts(
)['fields']['city'] if branch_facet else temp.facet_counts(
)['fields']['city']
elif branch_facet:
context['facets']['fields']['city'] = sqs.filter(
branch_exact=branch_facet).facet_counts()['fields']['city']
if branch_facet:
sqs = sqs.filter(branch_exact=branch_facet)
context['branch_facet_selected'] = branch_facet
else:
context['branch_facet_selected'] = ''
if year_facet:
sqs = sqs.filter(year_of_passing_exact__in=year_facet)
year_facet_string = None
for year in year_facet:
if not year_facet_string:
year_facet_string = str(year)
else:
year_facet_string = year_facet_string + "," + str(year)
context['year_facets_selected'] = year_facet_string
else:
context['year_facets_selected'] = ''
if city_facet:
sqs = sqs.filter(city_exact=city_facet)
context['city_facet_selected'] = city_facet
else:
context['city_facet_selected'] = ''
context['facets']['fields']['year_of_passing'] = self.facet_sorting(
context['facets']['fields']['year_of_passing'])
context['facets']['fields']['city'] = self.facet_sorting(
context['facets']['fields']['city'])
context['facets']['fields']['branch'] = self.facet_sorting(
context['facets']['fields']['branch'])
results = sqs.order_by('name')[0:INITIAL_RESULTS_COUNT]
resultcount = sqs.count()
context['resultcount'] = resultcount
context['initialoffset'] = INITIAL_RESULTS_COUNT
context['results'] = results
return context
def search(self, query=None, *args, **kwargs):
"""
Uses haystack to query groups.
Returns a SearchQuerySet
"""
sqs = SearchQuerySet()
user = kwargs.get('user', None)
# check to see if there is impersonation
if hasattr(user, 'impersonated_user'):
if isinstance(user.impersonated_user, User):
user = user.impersonated_user
is_a_superuser = user.profile.is_superuser
if query:
sqs = sqs.auto_query(sqs.query.clean(query))
if user:
if not is_a_superuser:
if not user.is_anonymous:
# if b/w superuser and anon
# (status+status_detail+(anon OR user)) OR (who_can_view__exact)
anon_query = Q(**{
'allow_anonymous_view': True,
})
user_query = Q(**{
'allow_user_view': True,
})
sec1_query = Q(
**{
'show_as_option': 1,
'status': 1,
'status_detail': 'active',
})
sec2_query = Q(
**{'who_can_view__exact': user.username})
query = reduce(operator.or_, [anon_query, user_query])
query = reduce(operator.and_, [sec1_query, query])
query = reduce(operator.or_, [query, sec2_query])
sqs = sqs.filter(query)
else: # anonymous
query = Q(
**{
'allow_anonymous_view': True,
'show_as_option': 1,
'status': 1,
'status_detail': 'active',
})
sqs = sqs.filter(query)
else: # anonymous
query = Q(
**{
'allow_anonymous_view': True,
'show_as_option': 1,
'status': 1,
'status_detail': 'active',
})
sqs = sqs.filter(query)
else:
if user:
if is_a_superuser:
# this is no-op. the .all().exclude(type='membership').models(Group) wouldn't work
#sqs = sqs.all()
sqs = sqs.filter(status=True)
else:
if not user.is_anonymous:
# (status+status_detail+anon OR who_can_view__exact)
anon_query = Q(**{
'allow_anonymous_view': True,
})
user_query = Q(**{
'allow_user_view': True,
})
sec1_query = Q(
**{
'show_as_option': 1,
'status': 1,
'status_detail': 'active',
})
sec2_query = Q(
**{'who_can_view__exact': user.username})
query = reduce(operator.or_, [anon_query, user_query])
query = reduce(operator.and_, [sec1_query, query])
query = reduce(operator.or_, [query, sec2_query])
sqs = sqs.filter(query)
else: # anonymous
query = Q(
**{
'allow_anonymous_view': True,
'show_as_option': 1,
'status': 1,
'status_detail': 'active',
})
sqs = sqs.filter(query)
else: # anonymous
query = Q(
**{
'allow_anonymous_view': True,
'show_as_option': 1,
'status': 1,
'status_detail': 'active',
})
sqs = sqs.filter(query)
sqs = sqs.order_by('-create_dt')
return sqs.models(self.model)
class LiveWhooshSearchQuerySetTestCase(TestCase):
def setUp(self):
super(LiveWhooshSearchQuerySetTestCase, self).setUp()
# Stow.
temp_path = os.path.join('tmp', 'test_whoosh_query')
self.old_whoosh_path = getattr(settings, 'HAYSTACK_WHOOSH_PATH',
temp_path)
settings.HAYSTACK_WHOOSH_PATH = temp_path
self.site = SearchSite()
self.sb = SearchBackend(site=self.site)
self.smmi = WhooshMockSearchIndex(MockModel, backend=self.sb)
self.site.register(MockModel, WhooshMockSearchIndex)
# Stow.
import haystack
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_site = haystack.site
haystack.site = self.site
self.sb.setup()
self.raw_whoosh = self.sb.index
self.parser = QueryParser(self.sb.content_field_name,
schema=self.sb.schema)
self.sb.delete_index()
self.sample_objs = []
for i in xrange(1, 4):
mock = MockModel()
mock.id = i
mock.author = 'daniel%s' % i
mock.pub_date = date(2009, 2, 25) - timedelta(days=i)
self.sample_objs.append(mock)
self.sq = SearchQuery(backend=self.sb)
self.sqs = SearchQuerySet(site=self.site)
def tearDown(self):
if os.path.exists(settings.HAYSTACK_WHOOSH_PATH):
shutil.rmtree(settings.HAYSTACK_WHOOSH_PATH)
settings.HAYSTACK_WHOOSH_PATH = self.old_whoosh_path
import haystack
haystack.site = self.old_site
settings.DEBUG = self.old_debug
super(LiveWhooshSearchQuerySetTestCase, self).tearDown()
def test_various_searchquerysets(self):
self.sb.update(self.smmi, self.sample_objs)
sqs = self.sqs.filter(content='Index')
self.assertEqual(sqs.query.build_query(), u'Index')
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!')
self.assertEqual(sqs.query.build_query(), u"'Indexed!'")
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!').filter(
pub_date__lte=date(2009, 8, 31))
self.assertEqual(sqs.query.build_query(),
u"('Indexed!' AND pub_date:[TO 20090831T000000])")
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!').filter(
pub_date__lte=date(2009, 2, 23))
self.assertEqual(sqs.query.build_query(),
u"('Indexed!' AND pub_date:[TO 20090223T000000])")
self.assertEqual(len(sqs), 2)
sqs = self.sqs.auto_query('Indexed!').filter(
pub_date__lte=date(2009, 2, 25)).filter(
django_id__in=[1, 2]).exclude(name='daniel1')
self.assertEqual(
sqs.query.build_query(),
u"('Indexed!' AND pub_date:[TO 20090225T000000] AND (django_id:\"1\" OR django_id:\"2\") AND NOT (name:daniel1))"
)
self.assertEqual(len(sqs), 1)
sqs = self.sqs.auto_query('re-inker')
self.assertEqual(sqs.query.build_query(), u"'re-inker'")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.auto_query('0.7 wire')
self.assertEqual(sqs.query.build_query(), u"('0.7' AND wire)")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.auto_query("daler-rowney pearlescent 'bell bronze'")
self.assertEqual(
sqs.query.build_query(),
u"(\"bell bronze\" AND 'daler-rowney' AND pearlescent)")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.models(MockModel)
self.assertEqual(sqs.query.build_query(), u'django_ct:core.mockmodel')
self.assertEqual(len(sqs), 3)
def test_all_regression(self):
sqs = SearchQuerySet()
self.assertEqual([result.pk for result in sqs], [])
self.sb.update(self.smmi, self.sample_objs)
self.assert_(self.sb.index.doc_count() > 0)
sqs = SearchQuerySet()
self.assertEqual(len(sqs), 3)
self.assertEqual(sorted([result.pk for result in sqs]),
[u'1', u'2', u'3'])
try:
sqs = repr(SearchQuerySet())
except:
self.fail()
def test_regression_space_query(self):
self.sb.update(self.smmi, self.sample_objs)
self.assert_(self.sb.index.doc_count() > 0)
sqs = SearchQuerySet().auto_query(" ")
self.assertEqual(len(sqs), 3)
sqs = SearchQuerySet().filter(content=" ")
self.assertEqual(len(sqs), 0)
def test_iter(self):
self.sb.update(self.smmi, self.sample_objs)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
sqs = self.sqs.auto_query('Indexed!')
results = [int(result.pk) for result in sqs]
self.assertEqual(sorted(results), [1, 2, 3])
self.assertEqual(len(backends.queries), 1)
def test_slice(self):
self.sb.update(self.smmi, self.sample_objs)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(sorted([int(result.pk) for result in results[1:3]]),
[1, 2])
self.assertEqual(len(backends.queries), 1)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(int(results[0].pk), 1)
self.assertEqual(len(backends.queries), 1)
def test_manual_iter(self):
self.sb.update(self.smmi, self.sample_objs)
results = self.sqs.auto_query('Indexed!')
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(sorted(results), [1, 2, 3])
self.assertEqual(len(backends.queries), 1)
def test_fill_cache(self):
self.sb.update(self.smmi, self.sample_objs)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(backends.queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 3)
self.assertEqual(len(backends.queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 3)
self.assertEqual(len(backends.queries), 2)
def test_cache_is_full(self):
self.sb.update(self.smmi, self.sample_objs)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.auto_query('Indexed!')
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(backends.queries), 1)
def test_count(self):
more_samples = []
for i in xrange(1, 50):
mock = MockModel()
mock.id = i
mock.author = 'daniel%s' % i
mock.pub_date = date(2009, 2, 25) - timedelta(days=i)
more_samples.append(mock)
self.sb.update(self.smmi, more_samples)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.sqs.all()
self.assertEqual(len(results), 49)
self.assertEqual(results._cache_is_full(), False)
self.assertEqual(len(backends.queries), 1)
class InterfaceTestCase(TestCase):
"""
Tests the interface of Xapian-Haystack.
Tests related to usability and expected behavior
go here.
"""
def setUp(self):
super(InterfaceTestCase, self).setUp()
types_names = ['book', 'magazine', 'article']
texts = ['This is a huge text',
'This is a medium text',
'This is a small text']
dates = [datetime.date(year=2010, month=1, day=1),
datetime.date(year=2010, month=2, day=1),
datetime.date(year=2010, month=3, day=1)]
summaries = ['This is a huge corrup\xe7\xe3o summary',
'This is a medium summary',
'This is a small summary']
for i in range(1, 13):
doc = Document()
doc.type_name = types_names[i % 3]
doc.number = i * 2
doc.name = "%s %d" % (doc.type_name, doc.number)
doc.date = dates[i % 3]
doc.summary = summaries[i % 3]
doc.text = texts[i % 3]
doc.save()
self.index = DocumentIndex()
self.ui = connections['default'].get_unified_index()
self.ui.build(indexes=[self.index])
self.backend = connections['default'].get_backend()
self.backend.update(self.index, Document.objects.all())
self.queryset = SearchQuerySet()
def tearDown(self):
Document.objects.all().delete()
#self.backend.clear()
super(InterfaceTestCase, self).tearDown()
def test_count(self):
self.assertEqual(self.queryset.count(), Document.objects.count())
def test_content_search(self):
result = self.queryset.filter(content='medium this')
self.assertEqual(sorted(pks(result)),
pks(Document.objects.all()))
# documents with "medium" AND "this" have higher score
self.assertEqual(pks(result)[:4], [1, 4, 7, 10])
def test_field_search(self):
self.assertEqual(pks(self.queryset.filter(name__contains='8')), [4])
self.assertEqual(pks(self.queryset.filter(type_name='book')),
pks(Document.objects.filter(type_name='book')))
self.assertEqual(pks(self.queryset.filter(text__contains='text huge')),
pks(Document.objects.filter(text__contains='text huge')))
def test_field_contains(self):
self.assertEqual(pks(self.queryset.filter(summary__contains='huge')),
pks(Document.objects.filter(summary__contains='huge')))
result = self.queryset.filter(summary__contains='huge summary')
self.assertEqual(sorted(pks(result)),
pks(Document.objects.all()))
# documents with "huge" AND "summary" have higher score
self.assertEqual(pks(result)[:4], [3, 6, 9, 12])
def test_field_exact(self):
self.assertEqual(pks(self.queryset.filter(name__exact='8')), [])
self.assertEqual(pks(self.queryset.filter(name__exact='magazine 2')), [1])
def test_content_exact(self):
self.assertEqual(pks(self.queryset.filter(content__exact='huge')), [])
def test_content_and(self):
self.assertEqual(pks(self.queryset.filter(content='huge').filter(summary='medium')), [])
self.assertEqual(len(self.queryset.filter(content='huge this')), 12)
self.assertEqual(len(self.queryset.filter(content='huge this').filter(summary__contains='huge')), 4)
def test_content_or(self):
self.assertEqual(len(self.queryset.filter(content='huge medium')), 8)
self.assertEqual(len(self.queryset.filter(content='huge medium small')), 12)
def test_field_and(self):
self.assertEqual(pks(self.queryset.filter(name='8').filter(name='4')), [])
def test_field_or(self):
self.assertEqual(pks(self.queryset.filter(name__contains='8 4')), [2, 4])
def test_field_in(self):
self.assertEqual(set(pks(self.queryset.filter(name__in=['magazine 2', 'article 4']))),
set(pks(Document.objects.filter(name__in=['magazine 2', 'article 4']))))
self.assertEqual(pks(self.queryset.filter(number__in=[4])),
pks(Document.objects.filter(number__in=[4])))
self.assertEqual(pks(self.queryset.filter(number__in=[4, 8])),
pks(Document.objects.filter(number__in=[4, 8])))
def test_private_fields(self):
self.assertEqual(pks(self.queryset.filter(django_id=4)),
pks(Document.objects.filter(id__in=[4])))
self.assertEqual(pks(self.queryset.filter(django_id__in=[2, 4])),
pks(Document.objects.filter(id__in=[2, 4])))
self.assertEqual(set(pks(self.queryset.models(Document))),
set(pks(Document.objects.all())))
def test_field_startswith(self):
self.assertEqual(len(self.queryset.filter(name__startswith='magaz')), 4)
self.assertEqual(set(pks(self.queryset.filter(summary__startswith='This is a huge'))),
set(pks(Document.objects.filter(summary__startswith='This is a huge'))))
def test_auto_query(self):
# todo: improve to query text only.
self.assertEqual(set(pks(self.queryset.auto_query("huge OR medium"))),
set(pks(Document.objects.filter(Q(text__contains="huge") |
Q(text__contains="medium")))))
self.assertEqual(set(pks(self.queryset.auto_query("huge AND medium"))),
set(pks(Document.objects.filter(Q(text__contains="huge") &
Q(text__contains="medium")))))
self.assertEqual(set(pks(self.queryset.auto_query("text:huge text:-this"))),
set(pks(Document.objects.filter(Q(text__contains="huge") &
~Q(text__contains="this")))))
self.assertEqual(len(self.queryset.filter(name=AutoQuery("8 OR 4"))), 2)
self.assertEqual(len(self.queryset.filter(name=AutoQuery("8 AND 4"))), 0)
def test_value_range(self):
self.assertEqual(set(pks(self.queryset.filter(number__lt=3))),
set(pks(Document.objects.filter(number__lt=3))))
self.assertEqual(set(pks(self.queryset.filter(django_id__gte=6))),
set(pks(Document.objects.filter(id__gte=6))))
def test_date_range(self):
date = datetime.date(year=2010, month=2, day=1)
self.assertEqual(set(pks(self.queryset.filter(date__gte=date))),
set(pks(Document.objects.filter(date__gte=date))))
date = datetime.date(year=2010, month=3, day=1)
self.assertEqual(set(pks(self.queryset.filter(date__lte=date))),
set(pks(Document.objects.filter(date__lte=date))))
def test_order_by(self):
# private order
self.assertEqual(pks(self.queryset.order_by("-django_id")),
pks(Document.objects.order_by("-id")))
# value order
self.assertEqual(pks(self.queryset.order_by("number")),
pks(Document.objects.order_by("number")))
# text order
self.assertEqual(pks(self.queryset.order_by("summary")),
pks(Document.objects.order_by("summary")))
# date order
self.assertEqual(pks(self.queryset.order_by("-date")),
pks(Document.objects.order_by("-date")))
def test_non_ascii_search(self):
"""
Regression test for #119.
"""
self.assertEqual(pks(self.queryset.filter(content='corrup\xe7\xe3o')),
pks(Document.objects.filter(summary__contains='corrup\xe7\xe3o')))
def test_multi_values_exact_search(self):
"""
Regression test for #103
"""
self.assertEqual(len(self.queryset.filter(tags__exact='tag')), 12)
self.assertEqual(len(self.queryset.filter(tags__exact='tag-test')), 8)
self.assertEqual(len(self.queryset.filter(tags__exact='tag-test-test')), 4)
class SearchQuerySetTestCase(TestCase):
fixtures = ["base_data.json", "bulk_data.json"]
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections["default"]._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.uuidmmsi = SimpleMockUUIDModelIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi, self.uuidmmsi])
connections["default"]._index = self.ui
# Update the "index".
backend = connections["default"].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections["default"]._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object"
r" at 0x[0-9A-Fa-f]+>, using=None>$",
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in iter(msqs)]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections["default"].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(
[int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]],
)
self.assertEqual(len(connections["default"].queries), 1)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections["default"].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(
check,
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
],
)
self.assertEqual(len(connections["default"].queries), 3)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
# CharPK testing
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections["default"]._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, ["sometext", "1234"])
self.assertEqual(len(connections["default"].queries), 1)
# UUID testing
self.ui.build(indexes=[self.uuidmmsi])
connections["default"]._index = self.ui
self.uuidmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(
loaded,
[
"53554c58-7051-4350-bcc9-dad75eb248a9",
"77554c58-7051-4350-bcc9-dad75eb24888",
],
)
connections["default"]._index = old_ui
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = list(results)
self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections["default"].queries), 4)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue("foo" in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections["default"]._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections["default"]._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost("foo", 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling_override(self):
sqs = self.msqs.filter(content="not the spellchecking query")
self.assertEqual(sqs.query.spelling_query, None)
sqs = self.msqs.set_spelling_query("override")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.spelling_query, "override")
def test_spelling_suggestions(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content="Indx")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion("indexy"), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search("foo")), 23)
self.assertEqual(
len(
self.msqs.raw_search(
"(content__exact:hello AND content__exact:world)")),
23,
)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend("charpk")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# Models with uuid primary keys.
sqs = SearchQuerySet()
sqs.query.backend = UUIDMockSearchBackend("uuid")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# If nothing is handled, you get nothing.
old_ui = connections["default"]._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections["default"]._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections["default"]._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections["default"]._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections["default"]._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# Restore.
connections["default"]._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query("test search -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND content__content=test search -stuff>",
)
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search -stuff>',
)
sqs = self.msqs.auto_query(
"test \"my thing\" search 'moar quotes' -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>",
)
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>',
)
sqs = self.msqs.auto_query("test - stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND content__content=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__content="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query("test search -stuff", fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND title__content=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff',
fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND title__content=test "my thing" search -stuff>',
)
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet("foo").facet("bar")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="smarblaph",
)
self.fail()
except FacetingError as e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.",
)
sqs = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="month",
)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="month",
).date_facet(
"bar",
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="year",
)
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet("foo", "[bar TO *]")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet(
"bar", "[100 TO 499]")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = (self.msqs.query_facet("foo", "[bar TO *]").query_facet(
"bar", "[100 TO 499]").query_facet("foo", "[1000 TO 1499]"))
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet("foo", "bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz")
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow("foo:moof")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo="bar", foo__lt="10")
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using="default")
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, "default")
def test_chaining(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content="bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter(content="oof")
sqs2 = self.msqs.filter(content="bar").filter(content="rab")
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, "OR")
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter_or(content="oof")
sqs2 = self.msqs.filter(content="bar").filter_or(content="rab")
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, "AND")
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def test_index_template(self):
r = SearchQuerySet().all()
self.assertTrue(r.count())
self.assertEquals(r.filter(text="bicicletas").count(), 1)
def _query_results(query, person):
"""
Actually build the query results for this person.
Make sure any result.content_type values are reflected in RESULT_TYPE_DISPLAY for display to the user.
"""
if len(query) < 2:
return []
query = query.replace('@sfu.ca', '') # hack to make email addresses searchable as userids
query = Clean(query)
# offerings person was a member of (coredata.CourseOffering)
if person:
members = Member.objects.filter(person=person).exclude(role='DROP').select_related('offering')
offering_slugs = set(m.offering.slug for m in members)
offering_results = SearchQuerySet().models(CourseOffering).filter(text=query) # offerings that match the query
offering_results = offering_results.filter(slug__in=offering_slugs) # ... and this person was in
else:
members = []
offering_results = []
# pages this person can view (pages.Page)
page_acl = set(['ALL'])
for m in members:
# builds a set of offering_slug+"_"+acl_value strings, which will match the permission_key field in the index
member_acl = set("%s_%s" % (m.offering.slug, acl) for acl in ACL_ROLES[m.role] if acl != 'ALL')
page_acl |= member_acl
page_results = SearchQuerySet().models(Page).filter(text=query) # pages that match the query
page_results = page_results.filter(permission_key__in=page_acl) # ... and are visible to this user
# discussion this person can view (discussion.DiscussionTopic)
if person:
discuss_results = SearchQuerySet().models(DiscussionTopic).filter(text=query) # discussions that match the query
discuss_results = discuss_results.filter(slug__in=offering_slugs) # ... and this person was in
else:
discuss_results = []
# students taught by instructor (coredata.Member)
instr_members = Member.objects.filter(person=person, role__in=['INST','TA']).exclude(offering__component='CAN') \
.select_related('offering')
if person and instr_members:
offering_slugs = set(m.offering.slug for m in instr_members)
member_results = SearchQuerySet().models(Member).filter(text=query) # members that match the query
member_results = member_results.filter(offering_slug__in=offering_slugs) # ... and this person was the instructor for
member_results = member_results.load_all()
else:
member_results = []
# combine and limit to best results
results = itertools.chain(
offering_results[:MAX_RESULTS],
page_results[:MAX_RESULTS],
member_results[:MAX_RESULTS],
discuss_results[:MAX_RESULTS],
)
results = (r for r in results if r is not None)
results = list(results)
results.sort(key=lambda result: -result.score)
results = results[:MAX_RESULTS] # (list before this could be n*MAX_RESULTS long)
return results
class LiveSolrSearchQuerySetTestCase(TestCase):
"""Used to test actual implementation details of the SearchQuerySet."""
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSolrSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SolrMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
print 'Reloading data...'
lssqstc_all_loaded = True
# Wipe it clean.
clear_solr_index()
# Force indexing of the content.
self.smmi.update()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveSolrSearchQuerySetTestCase, self).tearDown()
def test_load_all(self):
sqs = self.sqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(
sqs[0].object.foo,
u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]],
[2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(int(results[21].pk), 22)
self.assertEqual(len(connections['default'].queries), 1)
def test_count(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
self.assertEqual(sqs.count(), 23)
self.assertEqual(sqs.count(), 23)
self.assertEqual(len(sqs), 23)
self.assertEqual(sqs.count(), 23)
# Should only execute one query to count the length of the result set.
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.sqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test___and__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')
# Now for something more complex...
sqs3 = self.sqs.exclude(
title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar')
sqs = sqs3 & sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 3)
self.assertEqual(sqs.query.build_query(),
u'(NOT (title:moof) AND (foo OR baz) AND bar)')
def test___or__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')
# Now for something more complex...
sqs3 = self.sqs.exclude(
title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar').models(MockModel)
sqs = sqs3 | sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(),
u'((NOT (title:moof) AND (foo OR baz)) OR bar)')
def test_auto_query(self):
# Ensure bits in exact matches get escaped properly as well.
# This will break horrifically if escaping isn't working.
sqs = self.sqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains="pants:rule">')
self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"')
self.assertEqual(len(sqs), 0)
# Regressions
def test_regression_proper_start_offsets(self):
sqs = self.sqs.filter(text='index')
self.assertNotEqual(sqs.count(), 0)
id_counts = {}
for item in sqs:
if item.id in id_counts:
id_counts[item.id] += 1
else:
id_counts[item.id] = 1
for key, value in id_counts.items():
if value > 1:
self.fail(
"Result with id '%s' seen more than once in the results." %
key)
def test_regression_raw_search_breaks_slicing(self):
sqs = self.sqs.raw_search('text: index')
page_1 = [result.pk for result in sqs[0:10]]
page_2 = [result.pk for result in sqs[10:20]]
for pk in page_2:
if pk in page_1:
self.fail(
"Result with id '%s' seen more than once in the results." %
pk)
# RelatedSearchQuerySet Tests
def test_related_load_all(self):
sqs = self.rsqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(
sqs[0].object.foo,
u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
)
def test_related_load_all_queryset(self):
sqs = self.rsqs.load_all()
self.assertEqual(len(sqs._load_all_querysets), 0)
sqs = sqs.load_all_queryset(MockModel,
MockModel.objects.filter(id__gt=1))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], range(2, 24))
sqs = sqs.load_all_queryset(MockModel,
MockModel.objects.filter(id__gt=10))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])
def test_related_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.rsqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]],
[2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(int(results[21].pk), 22)
self.assertEqual(len(connections['default'].queries), 4)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[20:30]],
[21, 22, 23])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_manual_iter(self):
results = self.rsqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_related_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.rsqs._cache_is_full(), False)
results = self.rsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 5)
def test_quotes_regression(self):
sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
# Should not have empty terms.
self.assertEqual(sqs.query.build_query(),
u"44\xb048'40''N 20\xb028'32''E")
# Should not cause Solr to 500.
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing')
self.assertEqual(sqs.query.build_query(), u'blazing')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing saddles')
self.assertEqual(sqs.query.build_query(), u'blazing saddles')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles')
self.assertEqual(sqs.query.build_query(), u'\\"blazing saddles')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'"blazing saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'saddles"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing \'saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
self.assertEqual(sqs.query.build_query(),
u'mel "blazing \'\'saddles" \'')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
self.assertEqual(sqs.query.build_query(),
u'mel "blazing \'\'saddles" \'\\"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel')
self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
self.assertEqual(sqs.query.build_query(),
u'"blazing saddles" mel brooks')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
self.assertEqual(sqs.query.build_query(),
u'mel "blazing saddles" brooks')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
self.assertEqual(sqs.query.build_query(),
u'mel "blazing saddles" \\"brooks')
self.assertEqual(sqs.count(), 0)
def test_result_class(self):
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
def queryset(self, postdata, exclude=None):
"""
Build the `self.model` queryset limited to selected filters.
"""
if self.qs is None:
#qs = self.model.objects.all().select_related()
from haystack.query import SearchQuerySet
qs = SearchQuerySet()
if self.connection: qs = qs.using(self.connection)
qs = qs.filter(indexed_model_name__in=[self.model.__name__],
**self.global_filters)
else:
qs = self.qs
filters = {}
filters2 = []
# Then for each filter
for option in self.options:
# If filter is not excluded explicitly (used to get counts
# for choices).
if option == exclude: continue
# If filter contains valid data, check jQuery style encoding of array params
if option.field_name not in postdata and option.field_name + "[]" not in postdata:
continue
# Do filtering
if option.filter is not None:
qs_ = option.filter(qs, postdata)
if isinstance(qs_, dict):
filters.update(qs_)
else:
qs = qs_
else:
values = postdata.getlist(
option.field_name) + postdata.getlist(option.field_name +
"[]")
if option.type == "text":
# For full-text searching, don't use __in so that the search
# backend does its usual query operation.
values = " ".join(
values
) # should not really be more than one, but in case the parameter is specified multiple times in the query string
if self.qs is None:
# This is a Haystack search. Handle text a little differently.
# Wrap it in an AutoQuery so advanced search options like quoted phrases are used.
# Query both text and text_boosted, which has a boost applied at the field level.
from haystack.query import SQ
from haystack.inputs import AutoQuery
values = AutoQuery(values)
filters2.append(
SQ(text=values) | SQ(text_boosted=values))
else:
filters[option.orm_field_name] = values
elif not u'__ALL__' in values:
# if __ALL__ value presents in filter values
# then do not limit queryset
def parse_booleans(x):
for y in x:
if y in ("true", "on"):
yield True
elif y == "false":
yield False
else:
yield y
values = list(parse_booleans(values))
filters['%s__in' % option.orm_field_name] = values
# apply filters simultaneously so that filters on related objects are applied
# to the same related object. if they were applied chained (i.e. filter().filter())
# then they could apply to different objects.
if len(filters) + len(filters2) > 0:
qs = qs.filter(*filters2, **filters)
for name, key, default, func in self.sort_options:
if postdata.get("sort", "") == key:
if not func:
qs = qs.order_by(key)
else:
qs = func(qs)
# Django ORM but not Haystack
if hasattr(qs, 'distinct'):
return qs.distinct()
# Haystack but not Django ORM
else:
# Revise the SearchQuerySet to iterate over model objects
# rather than SearchResult objects.
class SR:
def __init__(self, qs):
self.qs = qs
def facet(self, field, **kwargs):
return self.qs.facet(field, **kwargs)
def count(self):
return len(self.qs)
def order_by(self, field):
return SR(self.qs.order_by(field))
def __len__(self):
return len(self.qs)
def __getitem__(self, index): # slices too
return SR(self.qs[index])
def __iter__(self):
for item in self.qs:
yield item.object
return SR(qs)
class LiveSolrMoreLikeThisTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSolrMoreLikeThisTestCase, self).setUp()
# Wipe it clean.
clear_solr_index()
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SolrMockModelSearchIndex()
self.sammi = SolrAnotherMockModelSearchIndex()
self.ui.build(indexes=[self.smmi, self.sammi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.smmi.update()
self.sammi.update()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
super(LiveSolrMoreLikeThisTestCase, self).tearDown()
def test_more_like_this(self):
mlt = self.sqs.more_like_this(MockModel.objects.get(pk=1))
self.assertEqual(mlt.count(), 22)
self.assertEqual([result.pk for result in mlt], [
'14', '6', '10', '22', '4', '5', '3', '12', '2', '19', '18', '13',
'15', '21', '7', '23', '20', '9', '1', '2', '17', '16'
])
self.assertEqual(len([result.pk for result in mlt]), 22)
alt_mlt = self.sqs.filter(name='daniel3').more_like_this(
MockModel.objects.get(pk=3))
self.assertEqual(alt_mlt.count(), 8)
self.assertEqual([result.pk for result in alt_mlt],
['17', '16', '19', '23', '22', '13', '1', '2'])
self.assertEqual(len([result.pk for result in alt_mlt]), 8)
alt_mlt_with_models = self.sqs.models(MockModel).more_like_this(
MockModel.objects.get(pk=1))
self.assertEqual(alt_mlt_with_models.count(), 20)
self.assertEqual([result.pk for result in alt_mlt_with_models], [
'14', '6', '10', '22', '4', '5', '3', '12', '2', '19', '18', '13',
'15', '21', '7', '23', '20', '9', '17', '16'
])
self.assertEqual(len([result.pk for result in alt_mlt_with_models]),
20)
if hasattr(MockModel.objects, 'defer'):
# Make sure MLT works with deferred bits.
mi = MockModel.objects.defer('foo').get(pk=1)
self.assertEqual(mi._deferred, True)
deferred = self.sqs.models(MockModel).more_like_this(mi)
self.assertEqual(deferred.count(), 0)
self.assertEqual([result.pk for result in deferred], [])
self.assertEqual(len([result.pk for result in deferred]), 0)
# Ensure that swapping the ``result_class`` works.
self.assertTrue(
isinstance(
self.sqs.result_class(MockSearchResult).more_like_this(
MockModel.objects.get(pk=1))[0], MockSearchResult))
def search(self, order_by='newest'):
self.clean()
sqs = SearchQuerySet()
if self.is_valid():
query = self.cleaned_data['q']
else:
query = ''
if not query:
return sqs.none()
if not order_by:
order_by = 'newest'
sqs = sqs.filter(status=True)
user = self.user
is_an_admin = user.profile.is_superuser
# making a special case for corp memb because it needs to utilize two settings
# (anonymoussearchcorporatemembers and membersearchcorporatemembers)
if CorpMemb and self.cleaned_data['models'] == ["%s.%s" % (CorpMemb._meta.app_label,
CorpMemb._meta.model_name)]:
filter_and, filter_or = CorpMemb.get_search_filter(user)
q_obj = None
if filter_and:
q_obj = Q(**filter_and)
if filter_or:
q_obj_or = reduce(operator.or_, [Q(**{key: value}) for key, value in filter_or.items()])
if q_obj:
q_obj = reduce(operator.and_, [q_obj, q_obj_or])
else:
q_obj = q_obj_or
if q_obj:
sqs = sqs.filter(q_obj)
if query:
sqs = sqs.auto_query(sqs.query.clean(query))
else:
if query:
sqs = sqs.auto_query(sqs.query.clean(query))
if user:
if not is_an_admin:
if not user.is_anonymous:
# if b/w admin and anon
# (status+status_detail+(anon OR user)) OR (who_can_view__exact)
anon_query = Q(**{'allow_anonymous_view':True,})
user_query = Q(**{'allow_user_view':True,})
sec1_query = Q(**{
'status_detail':'active',
})
sec2_query = Q(**{
'owner__exact':user.username
})
query = reduce(operator.or_, [anon_query, user_query])
query = reduce(operator.and_, [sec1_query, query])
query = reduce(operator.or_, [query, sec2_query])
sqs = sqs.filter(query)
else:
# if anonymous
sqs = sqs.filter(status_detail='active')
sqs = sqs.filter(allow_anonymous_view=True)
else:
# if anonymous
sqs = sqs.filter(status_detail='active')
sqs = sqs.filter(allow_anonymous_view=True)
else:
if user:
if is_an_admin:
sqs = sqs.all()
else:
if not user.is_anonymous:
# (status+status_detail+anon OR who_can_view__exact)
sec1_query = Q(**{
'status_detail':'active',
'allow_anonymous_view':True,
})
sec2_query = Q(**{
'owner__exact':user.username
})
query = reduce(operator.or_, [sec1_query, sec2_query])
sqs = sqs.filter(query)
else:
# if anonymous
sqs = sqs.filter(status_detail='active')
sqs = sqs.filter(allow_anonymous_view=True)
else:
# if anonymous
sqs = sqs.filter(status_detail='active')
sqs = sqs.filter(allow_anonymous_view=True)
# for solr,
# order_by can only called once so we have to do it here
if order_by == 'newest':
# changed order_by from create_dt to order because the order field
# is a common field in search indexes and used by other modules(news)
sqs = sqs.order_by('-order')
else:
sqs = sqs.order_by('order')
if self.load_all:
sqs = sqs.load_all()
return sqs
def _search_in_dictionary_language(
self, text: str, query_set: SearchQuerySet
) -> SearchQuerySet:
return query_set.filter(head_simplified__startswith=to_search_form(text))
class LiveWhooshRamStorageTestCase(TestCase):
def setUp(self):
super(LiveWhooshRamStorageTestCase, self).setUp()
# Stow.
self.old_whoosh_storage = settings.HAYSTACK_CONNECTIONS['whoosh'].get(
'STORAGE', 'file')
settings.HAYSTACK_CONNECTIONS['whoosh']['STORAGE'] = 'ram'
self.old_ui = connections['whoosh'].get_unified_index()
self.ui = UnifiedIndex()
self.wrtsi = WhooshRoundTripSearchIndex()
self.ui.build(indexes=[self.wrtsi])
self.sb = connections['whoosh'].get_backend()
connections['whoosh']._index = self.ui
# Stow.
import haystack
self.sb.setup()
self.raw_whoosh = self.sb.index
self.parser = QueryParser(self.sb.content_field_name,
schema=self.sb.schema)
self.sqs = SearchQuerySet('whoosh')
# Wipe it clean.
self.sqs.query.backend.clear()
# Fake indexing.
mock = MockModel()
mock.id = 1
self.sb.update(self.wrtsi, [mock])
def tearDown(self):
self.sqs.query.backend.clear()
settings.HAYSTACK_CONNECTIONS['whoosh'][
'STORAGE'] = self.old_whoosh_storage
connections['whoosh']._index = self.old_ui
super(LiveWhooshRamStorageTestCase, self).tearDown()
def test_ram_storage(self):
results = self.sqs.filter(id='core.mockmodel.1')
# Sanity check.
self.assertEqual(results.count(), 1)
# Check the individual fields.
result = results[0]
self.assertEqual(result.id, 'core.mockmodel.1')
self.assertEqual(result.text, 'This is some example text.')
self.assertEqual(result.name, 'Mister Pants')
self.assertEqual(result.is_active, True)
self.assertEqual(result.post_count, 25)
self.assertEqual(result.average_rating, 3.6)
self.assertEqual(result.pub_date, datetime(2009, 11, 21, 0, 0))
self.assertEqual(result.created, datetime(2009, 11, 21, 21, 31, 00))
self.assertEqual(result.tags,
['staff', 'outdoor', 'activist', 'scientist'])
self.assertEqual(result.sites, [u'3', u'5', u'1'])
self.assertEqual(result.empty_list, [])
def search(self):
# First, store the SearchQuerySet received from other processing.
if not self.is_valid():
print("Here")
return self.no_query_found()
print("Search Valid")
if self.cleaned_data["q"] == "":
# sqs = SearchQuerySet().exclude(no_such_field="x")
# sqs = SearchQuerySet().filter(content="soccer")
sqs = SearchQuerySet()
print(sqs.count())
# sqs = sqs.filter(content="soccer")
print("q = ''")
else:
print("q = something")
sqs = super(grasaSearchForm, self).search()
print(sqs.count())
selectedActivities = self.cleaned_data["activities"]
for activity in activityList:
for selectedActivity in selectedActivities:
print(selectedActivity)
if activity[0] == selectedActivity:
sqs = sqs.filter(content=activity[0])
selectedTransportations = self.cleaned_data["transportations"]
for transportation in transportationList:
for selectedTransportation in selectedTransportations:
if transportation[0] == selectedTransportation:
print('inside transport filter')
print(transportation[0])
print(selectedTransportation)
sqs = sqs.filter(content=transportation[0])
selectedGrades = self.cleaned_data["grades"]
for grade in gradesList:
for selectedGrade in selectedGrades:
if grade[0] == selectedGrade:
sqs = sqs.filter(content=grade[0])
selectedGenders = self.cleaned_data["genders"]
for gender in genderList:
for selectedGender in selectedGenders:
if gender[0] == selectedGender:
sqs = sqs.filter(content=gender[0])
# Needs changes
selectedFees = self.cleaned_data["fees"]
for selectedFee in selectedFees:
# These need to be specially written since they aren't stored as a range in the DB
if "Free" == selectedFee:
print("Free selected")
sqs = sqs.filter(fees=0.00)
if "$1-$25" == selectedFee:
print("$1-$25 selected")
sqs = sqs.filter(fees__gte=0.01)
sqs = sqs.filter(fees__lte=25.99)
if "$26-$50" == selectedFee:
print("$26-$50 selected")
print(sqs.count())
sqs = sqs.filter(fees__gte=26.00)
sqs = sqs.filter(fees__lte=50.99)
print(sqs.count())
if "$51-$75" == selectedFee:
print("$51-$75 selected")
sqs = sqs.filter(fees__gte=51.00)
sqs = sqs.filter(fees__lte=75.99)
if "$75+" == selectedFee:
sqs = sqs.filter(fees__gte=75.00)
selectedTimings = self.cleaned_data["timings"]
for timing in timingList:
for selectedTiming in selectedTimings:
if timing[0] == selectedTiming:
sqs = sqs.filter(content=timing[0])
print(sqs.count())
return sqs
def list(self, request, *args, **kwargs):
# If the incoming language is not specified, go with the default.
self.lang_code = request.QUERY_PARAMS.get('language', LANGUAGES[0])
if self.lang_code not in LANGUAGES:
raise ParseError(
"Invalid language supplied. Supported languages: %s" %
','.join(LANGUAGES))
context = {}
specs = {
'only_fields': self.request.QUERY_PARAMS.get('only', None),
'include_fields': self.request.QUERY_PARAMS.get('include', None)
}
for key in specs.keys():
if specs[key]:
setattr(self, key, {})
fields = [
x.strip().split('.') for x in specs[key].split(',') if x
]
for f in fields:
try:
getattr(self, key).setdefault(f[0], []).append(f[1])
except IndexError:
logger.warning(
"Field '%s' given in unsupported non-dot-format: '%s'"
% (key, specs[key]))
else:
setattr(self, key, None)
input_val = request.QUERY_PARAMS.get('input', '').strip()
q_val = request.QUERY_PARAMS.get('q', '').strip()
if not input_val and not q_val:
raise ParseError(
"Supply search terms with 'q=' or autocomplete entry with 'input='"
)
if input_val and q_val:
raise ParseError("Supply either 'q' or 'input', not both")
old_language = translation.get_language()[:2]
translation.activate(self.lang_code)
queryset = SearchQuerySet()
if hasattr(request, 'accepted_media_type') and re.match(
KML_REGEXP, request.accepted_media_type):
queryset = queryset.models(Unit)
self.only_fields['unit'].extend(['street_address', 'www_url'])
if input_val:
queryset = (queryset.filter(autosuggest=input_val).filter_or(
autosuggest_extra_searchwords=input_val).filter_or(
autosuggest_exact__exact=input_val))
else:
queryset = (queryset.filter(text=AutoQuery(q_val)).filter_or(
extra_searchwords=q_val).filter_or(address=q_val))
if 'municipality' in request.QUERY_PARAMS:
val = request.QUERY_PARAMS['municipality'].lower().strip()
if len(val) > 0:
municipalities = val.split(',')
muni_q_objects = [
SQ(municipality=m.strip()) for m in municipalities
]
muni_q = muni_q_objects.pop()
for q in muni_q_objects:
muni_q |= q
queryset = queryset.filter(
SQ(muni_q | SQ(django_ct='services.service')
| SQ(django_ct='munigeo.address')))
service = request.QUERY_PARAMS.get('service')
if service:
services = service.split(',')
queryset = queryset.filter(django_ct='services.unit').filter(
services__in=services)
models = set()
types = request.QUERY_PARAMS.get('type', '').split(',')
for t in types:
if t == 'service':
models.add(Service)
elif t == 'unit':
models.add(Unit)
elif t == 'address':
models.add(Address)
if len(models) > 0:
queryset = queryset.models(*list(models))
only = getattr(self, 'only_fields') or {}
include = getattr(self, 'include_fields') or {}
Unit.search_objects.only_fields = only.get('unit')
Unit.search_objects.include_fields = include.get('unit')
self.object_list = queryset.load_all()
# Switch between paginated or standard style responses
page = self.paginate_queryset(self.object_list)
serializer = self.get_serializer(page, many=True)
resp = self.get_paginated_response(serializer.data)
translation.activate(old_language)
return resp
def regulation_results_page(self, request):
all_regs = Part.objects.order_by('part_number')
regs = validate_regs_list(request)
order = validate_order(request)
search_query = request.GET.get('q', '').strip()
payload = {
'search_query': search_query,
'results': [],
'total_results': 0,
'regs': regs,
'all_regs': [],
}
if not search_query or len(urllib.parse.unquote(search_query)) == 1:
self.results = payload
return TemplateResponse(request, self.get_template(request),
self.get_context(request))
sqs = SearchQuerySet().filter(content=search_query)
payload.update({
'all_regs': [{
'short_name':
reg.short_name,
'id':
reg.part_number,
'num_results':
sqs.filter(
part=reg.part_number).models(SectionParagraph).count(),
'selected':
reg.part_number in regs
} for reg in all_regs]
})
payload.update({
'total_count':
sum([reg['num_results'] for reg in payload['all_regs']])
})
if len(regs) == 1:
sqs = sqs.filter(part=regs[0])
elif regs:
sqs = sqs.filter(part__in=regs)
if order == 'regulation':
sqs = sqs.order_by('part', 'section_order')
sqs = sqs.highlight(pre_tags=['<strong>'],
post_tags=['</strong>']).models(SectionParagraph)
for hit in sqs:
try:
snippet = Markup(" ".join(hit.highlighted))
except TypeError as e:
logger.warning(
"Query string {} produced a TypeError: {}".format(
search_query, e))
continue
short_name = all_regs.get(part_number=hit.part).short_name
hit_payload = {
'id':
hit.paragraph_id,
'part':
hit.part,
'reg':
short_name,
'label':
hit.title,
'snippet':
snippet,
'url':
"{}{}/{}/#{}".format(self.parent().url, hit.part,
hit.section_label, hit.paragraph_id),
}
payload['results'].append(hit_payload)
payload.update({'current_count': sqs.count()})
self.results = payload
context = self.get_context(request)
num_results = validate_num_results(request)
paginator = Paginator(payload['results'], num_results)
page_number = validate_page_number(request, paginator)
paginated_page = paginator.page(page_number)
context.update({
'current_count':
payload['current_count'],
'total_count':
payload['total_count'],
'paginator':
paginator,
'current_page':
page_number,
'num_results':
num_results,
'order':
order,
'results':
paginated_page,
'show_filters':
any(reg['selected'] is True for reg in payload['all_regs'])
})
return TemplateResponse(request, self.get_template(request), context)
class SearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections['default']._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections['default']._index = self.ui
# Update the "index".
backend = connections['default'].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections['default']._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(
repr(self.msqs).replace("u'", "'"),
"[<SearchResult: core.mockmodel (pk='1')>, <SearchResult: core.mockmodel (pk='2')>, <SearchResult: core.mockmodel (pk='3')>, <SearchResult: core.mockmodel (pk='4')>, <SearchResult: core.mockmodel (pk='5')>, <SearchResult: core.mockmodel (pk='6')>, <SearchResult: core.mockmodel (pk='7')>, <SearchResult: core.mockmodel (pk='8')>, <SearchResult: core.mockmodel (pk='9')>, <SearchResult: core.mockmodel (pk='10')>, <SearchResult: core.mockmodel (pk='11')>, <SearchResult: core.mockmodel (pk='12')>, <SearchResult: core.mockmodel (pk='13')>, <SearchResult: core.mockmodel (pk='14')>, <SearchResult: core.mockmodel (pk='15')>, <SearchResult: core.mockmodel (pk='16')>, <SearchResult: core.mockmodel (pk='17')>, <SearchResult: core.mockmodel (pk='18')>, <SearchResult: core.mockmodel (pk='19')>, '...(remaining elements truncated)...']"
)
self.assertEqual(len(connections['default'].queries), 1)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in msqs]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(check, [
u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'10', u'11',
u'12', u'13', u'14', u'15', u'16', u'17', u'18', u'19', u'20',
u'21', u'22', u'23'
])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections['default']._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [u'sometext', u'1234'])
self.assertEqual(len(connections['default'].queries), 1)
connections['default']._index = old_ui
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
sqs = SearchQuerySet().all()
sqs.query.backend = MixedMockSearchBackend('default')
results = sqs
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 0)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [])
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 9)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10])
self.assertEqual(len(connections['default'].queries), 2)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 17)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20])
self.assertEqual(len(connections['default'].queries), 4)
results._fill_cache(20, 30)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [
1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22,
23
])
self.assertEqual(len(connections['default'].queries), 6)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue('foo' in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections['default']._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections['default']._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost('foo', 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content='Indx')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search('foo')), 23)
self.assertEqual(
len(
self.msqs.raw_search(
'(content__exact:hello AND content__exact:world)')), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend('charpk')
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# If nothing is handled, you get nothing.
old_ui = connections['default']._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections['default']._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections['default']._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections['default']._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections['default']._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# Restore.
connections['default']._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query('test search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains=test search -stuff>')
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search -stuff>')
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search \'moar quotes\' -stuff>'
)
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search \'moar quotes\' "foo -stuff>'
)
sqs = self.msqs.auto_query('test - stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND content__contains=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query('test search -stuff', fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND title__contains=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff',
fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND title__contains=test "my thing" search -stuff>')
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet('foo').facet('bar')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='smarblaph')
self.fail()
except FacetingError as e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second."
)
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month').date_facet(
'bar',
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='year')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet('foo', '[bar TO *]')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet('foo', '[bar TO *]').query_facet(
'bar', '[100 TO 499]')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = self.msqs.query_facet('foo', '[bar TO *]').query_facet(
'bar', '[100 TO 499]').query_facet('foo', '[1000 TO 1499]')
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet('foo', 'bar')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet('foo', 'bar').stats_facet('foo', 'baz')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet('foo', 'bar').stats_facet('moof', 'baz')
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow('foo:moof')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo='bar', foo__lt='10')
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using='default')
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, 'default')
def test_chaining(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content='bar')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content='foo')
sqs2 = self.msqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content='foo')
sqs2 = self.msqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content='foo').filter(content='oof')
sqs2 = self.msqs.filter(content='bar').filter(content='rab')
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, 'OR')
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content='foo').filter_or(content='oof')
sqs2 = self.msqs.filter(content='bar').filter_or(content='rab')
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, 'AND')
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
class LiveWhooshSearchQuerySetTestCase(WhooshTestCase):
def setUp(self):
super(LiveWhooshSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_ui = connections['whoosh'].get_unified_index()
self.ui = UnifiedIndex()
self.wmmi = WhooshMockSearchIndex()
self.ui.build(indexes=[self.wmmi])
self.sb = connections['whoosh'].get_backend()
connections['whoosh']._index = self.ui
self.sb.setup()
self.raw_whoosh = self.sb.index
self.parser = QueryParser(self.sb.content_field_name, schema=self.sb.schema)
self.sb.delete_index()
self.sample_objs = []
for i in range(1, 4):
mock = MockModel()
mock.id = i
mock.author = 'daniel%s' % i
mock.pub_date = date(2009, 2, 25) - timedelta(days=i)
self.sample_objs.append(mock)
self.sq = connections['whoosh'].get_query()
self.sqs = SearchQuerySet('whoosh')
def tearDown(self):
connections['whoosh']._index = self.old_ui
super(LiveWhooshSearchQuerySetTestCase, self).tearDown()
def test_various_searchquerysets(self):
self.sb.update(self.wmmi, self.sample_objs)
sqs = self.sqs.filter(content='Index')
self.assertEqual(sqs.query.build_query(), u'(Index)')
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!')
self.assertEqual(sqs.query.build_query(), u"('Indexed!')")
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!').filter(pub_date__lte=date(2009, 8, 31))
self.assertEqual(sqs.query.build_query(), u"(('Indexed!') AND pub_date:([to 20090831000000]))")
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!').filter(pub_date__lte=date(2009, 2, 23))
self.assertEqual(sqs.query.build_query(), u"(('Indexed!') AND pub_date:([to 20090223000000]))")
self.assertEqual(len(sqs), 2)
sqs = self.sqs.auto_query('Indexed!').filter(pub_date__lte=date(2009, 2, 25)).filter(django_id__in=[1, 2]).exclude(name='daniel1')
self.assertEqual(sqs.query.build_query(), u'((\'Indexed!\') AND pub_date:([to 20090225000000]) AND django_id:(1 OR 2) AND NOT (name:(daniel1)))')
self.assertEqual(len(sqs), 1)
sqs = self.sqs.auto_query('re-inker')
self.assertEqual(sqs.query.build_query(), u"('re-inker')")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.auto_query('0.7 wire')
self.assertEqual(sqs.query.build_query(), u"('0.7' wire)")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.auto_query("daler-rowney pearlescent 'bell bronze'")
self.assertEqual(sqs.query.build_query(), u"('daler-rowney' pearlescent 'bell bronze')")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.models(MockModel)
self.assertEqual(sqs.query.build_query(), u'*')
self.assertEqual(len(sqs), 3)
def test_all_regression(self):
sqs = SearchQuerySet('whoosh')
self.assertEqual([result.pk for result in sqs], [])
self.sb.update(self.wmmi, self.sample_objs)
self.assertTrue(self.sb.index.doc_count() > 0)
sqs = SearchQuerySet('whoosh')
self.assertEqual(len(sqs), 3)
self.assertEqual(sorted([result.pk for result in sqs]), [u'1', u'2', u'3'])
try:
sqs = repr(SearchQuerySet('whoosh'))
except:
self.fail()
def test_regression_space_query(self):
self.sb.update(self.wmmi, self.sample_objs)
self.assertTrue(self.sb.index.doc_count() > 0)
sqs = SearchQuerySet('whoosh').auto_query(" ")
self.assertEqual(len(sqs), 3)
sqs = SearchQuerySet('whoosh').filter(content=" ")
self.assertEqual(len(sqs), 0)
def test_iter(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
sqs = self.sqs.auto_query('Indexed!')
results = [int(result.pk) for result in sqs]
self.assertEqual(sorted(results), [1, 2, 3])
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_slice(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(sorted([int(result.pk) for result in results[1:3]]), [1, 2])
self.assertEqual(len(connections['whoosh'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(int(results[0].pk), 1)
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_values_slicing(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
# TODO: this would be a good candidate for refactoring into a TestCase subclass shared across backends
# The values will come back as strings because Hasytack doesn't assume PKs are integers.
# We'll prepare this set once since we're going to query the same results in multiple ways:
expected_pks = ['3', '2', '1']
results = self.sqs.all().order_by('pub_date').values('pk')
self.assertListEqual([i['pk'] for i in results[1:11]], expected_pks)
results = self.sqs.all().order_by('pub_date').values_list('pk')
self.assertListEqual([i[0] for i in results[1:11]], expected_pks)
results = self.sqs.all().order_by('pub_date').values_list('pk', flat=True)
self.assertListEqual(results[1:11], expected_pks)
self.assertEqual(len(connections['whoosh'].queries), 3)
def test_manual_iter(self):
self.sb.update(self.wmmi, self.sample_objs)
results = self.sqs.auto_query('Indexed!')
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(sorted(results), [1, 2, 3])
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_fill_cache(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['whoosh'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 3)
self.assertEqual(len(connections['whoosh'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 3)
self.assertEqual(len(connections['whoosh'].queries), 2)
def test_cache_is_full(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.auto_query('Indexed!')
[result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_count(self):
more_samples = []
for i in range(1, 50):
mock = MockModel()
mock.id = i
mock.author = 'daniel%s' % i
mock.pub_date = date(2009, 2, 25) - timedelta(days=i)
more_samples.append(mock)
self.sb.update(self.wmmi, more_samples)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results), 49)
self.assertEqual(results._cache_is_full(), False)
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_query_generation(self):
sqs = self.sqs.filter(SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")))
self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))")
def test_result_class(self):
self.sb.update(self.wmmi, self.sample_objs)
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
def search(request):
# TODO: used forms in every search type
def _render_search_form(form=None):
return render(request, 'djangobb_forum/search_form.html', {
'categories': Category.objects.all(),
'form': form,
})
if not 'action' in request.GET:
return _render_search_form(form=PostSearchForm())
if request.GET.get("show_as") == "posts":
show_as_posts = True
template_name = 'djangobb_forum/search_posts.html'
else:
show_as_posts = False
template_name = 'djangobb_forum/search_topics.html'
context = {}
# Create 'user viewable' pre-filtered topics/posts querysets
viewable_category = Category.objects.all()
topics = Topic.objects.all().order_by("-last_post__created")
posts = Post.objects.all().order_by('-created')
user = request.user
if not user.is_superuser:
user_groups = user.groups.all() or [
] # need 'or []' for anonymous user otherwise: 'EmptyManager' object is not iterable
viewable_category = viewable_category.filter(
Q(groups__in=user_groups) | Q(groups__isnull=True))
topics = Topic.objects.filter(forum__category__in=viewable_category)
posts = Post.objects.filter(
topic__forum__category__in=viewable_category)
base_url = None
_generic_context = True
action = request.GET['action']
if action == 'show_24h':
date = timezone.now() - timedelta(days=1)
if show_as_posts:
context["posts"] = posts.filter(
Q(created__gte=date) | Q(updated__gte=date))
else:
context["topics"] = topics.filter(
Q(last_post__created__gte=date)
| Q(last_post__updated__gte=date))
_generic_context = False
elif action == 'show_new':
if not user.is_authenticated:
raise Http404(
"Search 'show_new' not available for anonymous user.")
try:
last_read = PostTracking.objects.get(user=user).last_read
except PostTracking.DoesNotExist:
last_read = None
if last_read:
if show_as_posts:
context["posts"] = posts.filter(
Q(created__gte=last_read) | Q(updated__gte=last_read))
else:
context["topics"] = topics.filter(
Q(last_post__created__gte=last_read)
| Q(last_post__updated__gte=last_read))
_generic_context = False
else:
#searching more than forum_settings.SEARCH_PAGE_SIZE in this way - not good idea :]
topics_id = [
topic.id for topic in topics[:forum_settings.SEARCH_PAGE_SIZE]
if forum_extras.has_unreads(topic, user)
]
topics = Topic.objects.filter(
id__in=topics_id) # to create QuerySet
elif action == 'show_unanswered':
topics = topics.filter(post_count=1)
elif action == 'show_subscriptions':
topics = topics.filter(subscribers__id=user.id)
elif action == 'show_user':
# Show all posts from user or topics started by user
if not user.is_authenticated:
raise Http404(
"Search 'show_user' not available for anonymous user.")
user_id = request.GET.get("user_id", user.id)
try:
user_id = int(user_id)
except ValueError:
raise SuspiciousOperation()
if user_id != user.id:
try:
search_user = User.objects.get(id=user_id)
except User.DoesNotExist:
messages.error(request, _("Error: User unknown!"))
return HttpResponseRedirect(request.path)
messages.info(
request,
_("Filter by user '%(username)s'.") %
{'username': search_user.username})
if show_as_posts:
posts = posts.filter(user__id=user_id)
else:
# show as topic
topics = topics.filter(posts__user__id=user_id).order_by(
"-last_post__created").distinct()
base_url = "?action=show_user&user_id=%s&show_as=" % user_id
elif action == 'search':
form = PostSearchForm(request.GET)
if not form.is_valid():
return _render_search_form(form)
keywords = form.cleaned_data['keywords']
author = form.cleaned_data['author']
forum = form.cleaned_data['forum']
search_in = form.cleaned_data['search_in']
sort_by = form.cleaned_data['sort_by']
sort_dir = form.cleaned_data['sort_dir']
query = SearchQuerySet().models(Post)
if author:
query = query.filter(author__username=author)
if forum != '0':
query = query.filter(forum__id=forum)
if keywords:
if search_in == 'all':
query = query.filter(SQ(topic=keywords) | SQ(text=keywords))
elif search_in == 'message':
query = query.filter(text=keywords)
elif search_in == 'topic':
query = query.filter(topic=keywords)
order = {
'0': 'created',
'1': 'author',
'2': 'topic',
'3': 'forum'
}.get(sort_by, 'created')
if sort_dir == 'DESC':
order = '-' + order
post_pks = query.values_list("pk", flat=True)
if not show_as_posts:
# TODO: We have here a problem to get a list of topics without double entries.
# Maybe we must add a search index over topics?
# Info: If whoosh backend used, setup HAYSTACK_ITERATOR_LOAD_PER_QUERY
# to a higher number to speed up
context["topics"] = topics.filter(posts__in=post_pks).distinct()
else:
# FIXME: How to use the pre-filtered query from above?
posts = posts.filter(pk__in=post_pks).order_by(order)
context["posts"] = posts
get_query_dict = request.GET.copy()
get_query_dict.pop("show_as")
base_url = "?%s&show_as=" % get_query_dict.urlencode()
_generic_context = False
if _generic_context:
if show_as_posts:
context["posts"] = posts.filter(
topic__in=topics).order_by('-created')
else:
context["topics"] = topics
if base_url is None:
base_url = "?action=%s&show_as=" % action
if show_as_posts:
context['posts_page'] = get_page(context['posts'], request,
forum_settings.SEARCH_PAGE_SIZE)
context["as_topic_url"] = base_url + "topics"
post_count = context["posts"].count()
messages.success(request, _("Found %i posts.") % post_count)
else:
context['topics_page'] = get_page(context['topics'], request,
forum_settings.SEARCH_PAGE_SIZE)
context["as_post_url"] = base_url + "posts"
topic_count = context["topics"].count()
messages.success(request, _("Found %i topics.") % topic_count)
return render(request, template_name, context)
def images_search(request):
"""
View to make search of images.
:param request: Request to serve
"""
query = ''
time_start = ''
time_end = ''
post = request.POST
enable_checkboxes = False
if request.method == 'GET' and request.GET.get('q') is not None:
request.method = 'POST'
enable_checkboxes = True
post = {
'q': request.GET['q'],
'time_start': request.GET.get('time_start', ''),
'time_end': request.GET.get('time_end', '')
}
if request.method == 'POST':
# Check if session have to be deleted
if post.get('clear_form'):
request.session.flush()
pass
# New search, clear data stored in session
results = None
request.session['results_images'] = None
form = SearchForm(post)
restart = post.get('restart') is not None
base_query = SearchQuerySet().models(Image).order_by('category_label')
if form.is_valid():
images = []
categories_to_search = []
query = form.cleaned_data['q'] if not restart else ''
# Get categories to search and get left menu data
for i in ImageCategory.objects.filter(visible_on_website=True):
category_images = {}
category_images["label_name"] = i.label
category_images["label_code"] = i.value
category_images["images"] = []
search_set = base_query.filter(
category_label__exact=i.label,
ready_to_go=True).order_by('date')
category_images["number_of_images"] = len(search_set)
if restart or enable_checkboxes or post.get("checkbox" +
str(i.value)):
categories_to_search.append(i.label)
images.append(category_images)
images = sorted(images, key=lambda k: k["label_name"])
time_start = post.get('time_start') if not restart else ''
time_end = post.get('time_end') if not restart else ''
# Options if query is provided
if query != "":
if time_start != "" and time_end != "":
results = \
base_query.filter(imgtext__icontains=query, ready_to_go=True,
category_label__in=categories_to_search,
date__gte=time_start,
date__lte=time_end).\
order_by('date')
elif time_start != "":
results = \
base_query.filter(imgtext__icontains=query, ready_to_go=True,
category_label__in=categories_to_search,
date__gte=time_start).\
order_by('date')
elif time_end != "":
results = \
base_query.filter(imgtext__icontains=query, ready_to_go=True,
category_label__in=categories_to_search,
date__lte=time_end).\
order_by('date')
else:
results = \
base_query.filter(imgtext__icontains=query, ready_to_go=True,
category_label__in=categories_to_search).\
order_by('date')
elif time_start != "" or time_end != "":
if time_start != "" and time_end != "":
results = \
base_query.filter(ready_to_go=True,
category_label__in=categories_to_search,
date__gte=time_start,
date__lte=time_end).\
order_by('date')
elif time_start != "":
results = \
base_query.filter(ready_to_go=True,
category_label__in=categories_to_search,
date__gte=time_start).\
order_by('date')
elif time_end != "":
results = \
base_query.filter(ready_to_go=True,
category_label__in=categories_to_search,
date__lte=time_end).\
order_by('date')
else:
if len(categories_to_search) == 1:
return HttpResponseRedirect(
reverse('resources:images-category',
kwargs={
'category': categories_to_search.pop()
}))
else:
results = base_query.all().filter(ready_to_go=True,
category_label__in=categories_to_search).\
order_by('date')
else:
if len(categories_to_search) > 1:
results = base_query.all().filter(ready_to_go=True,
category_label__in=categories_to_search).\
order_by('date')
elif len(categories_to_search) == 1:
return HttpResponseRedirect(
reverse(
'resources:images-category',
kwargs={'category': categories_to_search.pop()}))
if results is None:
results = base_query.all()
# Store results in session
request.session['results_images'] = results
request.session['images_images'] = images
request.session['enabled_categories'] = categories_to_search
request.session['query'] = query
request.session['time_start'] = time_start
request.session['time_end'] = time_end
else:
results = request.session.get('results_images')
images = request.session.get('images_images')
categorized = {
cat: sorted(g, key=lambda x: _(x.title))
for cat, g in groupby(results, key=lambda x: x.category_label)
}
return render(
request, 'resources/images-search-results.html', {
'results': results,
'images': images,
'query': request.session['query'],
'time_start': request.session['time_start'],
'time_end': request.session['time_end'],
'enabled_categories': request.session['enabled_categories'],
'categorized': categorized
})
def autocomplete(self, request, **kwargs):
# Note: If the parameter in question (e.g. query, limit, etc.)
# is not sent as part of the request, the request.GET.get()
# method will set it to None unless a default is specified.
query = request.GET.get('query', '')
organism_uri = request.GET.get('organism')
limit = request.GET.get('limit')
if limit:
try:
# Make sure the input is already an integer
# or can be coerced into one.
limit = int(limit)
except ValueError:
logger.error("'limit' parameter passed was specified "
"incorrectly.")
# Limit was specified incorrectly, so default to
# GENES_API_RESULT_LIMIT setting.
limit = GENES_API_RESULT_LIMIT
else:
limit = GENES_API_RESULT_LIMIT
# We want to sort results by three fields: First by search score,
# then by name length (standard_name if present, if not then
# systematic_name), and finally by this name's alphabetical order.
# We sort by gene name length because a long gene name and
# a short gene name can have the same score if they contain the n-gram.
# A user can always type more to get the long one, but they can't type
# less to get the short one. By returning the shortest first, we make
# sure that this choice is always available to the user, even if a
# limit is applied.
#
# *Note: We use SearchQuerySet's order_by() function to sort by these
# three fields, but this current implementation is Elasticsearch
# specific. (Elasticsearch uses '_score' while it is simply 'score' in
# Solr). At some point we might want to expand this to be compatible
# with other search engines, like Solr, Xapian and Whoosh.
# See Haystack issue here:
# https://github.com/django-haystack/django-haystack/issues/1431
sqs = SearchQuerySet().models(Gene).autocomplete(
wall_of_name_auto=query).order_by('-_score', 'name_length',
'wall_of_name_auto')
if organism_uri:
organism = OrganismResource().get_via_uri(organism_uri, request)
sqs = sqs.filter(organism=organism)
suggestions = []
# Get slice of sqs for 'limit' specified (or use full sqs if no limit
# was specified).
for result in sqs[:limit]:
gene = result.object
suggestions.append({
'id': gene.id,
'score': result.score,
'entrezid': gene.entrezid,
'standard_name': gene.standard_name,
'systematic_name': gene.systematic_name,
'description': gene.description
})
# Return a JSON object instead of an array, as returning an array
# could make the information vulnerable to an XSS attack.
response = {'results': suggestions}
return self.create_response(request, response)
class SearchTopicIndexTest(TestCase):
def setUp(self):
utils.cache_clear()
self.topic_sqs = SearchQuerySet().models(Topic)
self.now = timezone.now()
self.yesterday = timezone.now() - datetime.timedelta(days=1)
self.tomorrow = timezone.now() + datetime.timedelta(days=1)
def test_index_queryset_excludes_private_topics(self):
"""
index_queryset should exclude private topics
"""
utils.create_private_topic()
self.assertEqual(len(TopicIndex().index_queryset()), 0)
category = utils.create_category()
utils.create_topic(category)
self.assertEqual(len(TopicIndex().index_queryset()), 1)
def test_indexing_excludes_private_topics(self):
"""
rebuild_index command should exclude private topics
"""
utils.create_private_topic()
category = utils.create_category()
topic = utils.create_topic(category)
rebuild_index()
self.assertEqual([s.object for s in self.topic_sqs], [topic])
def test_indexing_is_removed(self):
"""
Should set the removed flag when either\
topic, subcategory or category are removed
"""
main_category = utils.create_category()
category = utils.create_category(parent=main_category)
topic = utils.create_topic(category)
rebuild_index()
self.assertEqual(
len(self.topic_sqs.filter(is_removed=False)), 1)
topic.is_removed = True
topic.save()
rebuild_index()
self.assertEqual(
len(self.topic_sqs.filter(is_removed=False)), 0)
category.is_removed = True
category.save()
topic.is_removed = False
topic.save()
rebuild_index()
self.assertEqual(
len(self.topic_sqs.filter(is_removed=False)), 0)
main_category.is_removed = True
main_category.save()
category.is_removed = False
category.save()
topic.is_removed = False
topic.save()
rebuild_index()
self.assertEqual(
len(self.topic_sqs.filter(is_removed=False)), 0)
main_category.is_removed = False
main_category.save()
rebuild_index()
self.assertEqual(
len(self.topic_sqs.filter(is_removed=False)), 1)
def test_indexing_slug_empty(self):
"""
Should store the slug as an empty string
"""
category = utils.create_category()
topic = utils.create_topic(category)
topic.slug = ''
topic.save()
rebuild_index()
self.assertEqual(len(self.topic_sqs.all()), 1)
self.assertEqual(
list(self.topic_sqs.all())[0]
.get_stored_fields()['slug'],
'')
def test_indexing_main_category_name(self):
"""
Should store the main category name
"""
main_category = utils.create_category()
category = utils.create_category(parent=main_category)
utils.create_topic(category)
rebuild_index()
self.assertEqual(len(self.topic_sqs.all()), 1)
self.assertEqual(
list(self.topic_sqs.all())[0]
.get_stored_fields()['main_category_name'],
main_category.title)
def test_indexing_text_include_comments(self):
"""
Should include topic title and all comments
"""
category = utils.create_category()
topic = utils.create_topic(category, title='my title')
utils.create_comment(topic=topic, comment_html='<span>foo</span>')
utils.create_comment(topic=topic, comment_html='<b>bar</b>')
rebuild_index()
self.assertEqual(len(self.topic_sqs.all()), 1)
self.assertEqual(
len(self.topic_sqs.filter(text='my title foo bar')), 1)
self.assertEqual(
len(self.topic_sqs.filter(text='bar')), 1)
self.assertEqual(
len(self.topic_sqs.filter(text='<b>')), 0)
self.assertEqual(
len(self.topic_sqs.filter(text='span')), 0)
def test_indexing_text_template(self):
"""
Should include topic title and all comments
"""
category = utils.create_category()
topic = utils.create_topic(category, title='my title')
utils.create_comment(topic=topic, comment_html='<span>foo</span>')
utils.create_comment(topic=topic, comment_html='<b>bar</b>')
self.assertEqual(
render_to_string(
'search/indexes/spirit_topic/topic_text.txt',
context={'object': topic}),
'my title\n\nbar\n\nfoo\n\n')
def test_indexing_build_queryset_by_topic(self):
"""
Should update topics based on modified times
"""
main_category = utils.create_category(
reindex_at=self.yesterday)
category = utils.create_category(
parent=main_category, reindex_at=self.yesterday)
topic = utils.create_topic(
category,
reindex_at=self.yesterday, last_active=self.yesterday)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 0)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 0)
topic.reindex_at = self.tomorrow
topic.save()
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.now)), 0)
def test_indexing_build_queryset_by_comment(self):
"""
Should update topics based on modified times
"""
main_category = utils.create_category(
reindex_at=self.yesterday)
category = utils.create_category(
parent=main_category, reindex_at=self.yesterday)
topic = utils.create_topic(
category,
reindex_at=self.yesterday, last_active=self.yesterday)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 0)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 0)
topic.last_active = self.tomorrow
topic.save()
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.now)), 0)
def test_indexing_build_queryset_by_category(self):
"""
Should update topics based on modified times
"""
main_category = utils.create_category(
reindex_at=self.yesterday)
category = utils.create_category(
parent=main_category, reindex_at=self.yesterday)
utils.create_topic(
category,
reindex_at=self.yesterday, last_active=self.yesterday)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 0)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 0)
category.reindex_at = self.tomorrow
category.save()
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.now)), 0)
def test_indexing_build_queryset_by_subcategory(self):
"""
Should update topics based on modified times
"""
main_category = utils.create_category(
reindex_at=self.yesterday)
category = utils.create_category(
parent=main_category, reindex_at=self.yesterday)
utils.create_topic(
category,
reindex_at=self.yesterday, last_active=self.yesterday)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 0)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 0)
main_category.reindex_at = self.tomorrow
main_category.save()
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.tomorrow)), 1)
self.assertEqual(
len(TopicIndex().build_queryset(start_date=self.now, end_date=self.now)), 0)
class LiveWhooshMoreLikeThisTestCase(WhooshTestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveWhooshMoreLikeThisTestCase, self).setUp()
# Stow.
self.old_ui = connections['whoosh'].get_unified_index()
self.ui = UnifiedIndex()
self.wmmi = WhooshMockSearchIndex()
self.wamsi = WhooshAnotherMockSearchIndex()
self.ui.build(indexes=[self.wmmi, self.wamsi])
self.sb = connections['whoosh'].get_backend()
connections['whoosh']._index = self.ui
self.sb.setup()
self.raw_whoosh = self.sb.index
self.parser = QueryParser(self.sb.content_field_name,
schema=self.sb.schema)
self.sb.delete_index()
self.wmmi.update()
self.wamsi.update()
self.sqs = SearchQuerySet('whoosh')
def tearDown(self):
connections['whoosh']._index = self.old_ui
super(LiveWhooshMoreLikeThisTestCase, self).tearDown()
# We expect failure here because, despite not changing the code, Whoosh
# 2.5.1 returns incorrect counts/results. Huzzah.
@unittest.expectedFailure
def test_more_like_this(self):
mlt = self.sqs.more_like_this(MockModel.objects.get(pk=22))
self.assertEqual(mlt.count(), 22)
self.assertEqual(
sorted([result.pk for result in mlt]),
sorted([
u'9', u'8', u'7', u'6', u'5', u'4', u'3', u'2', u'1', u'21',
u'20', u'19', u'18', u'17', u'16', u'15', u'14', u'13', u'12',
u'11', u'10', u'23'
]))
self.assertEqual(len([result.pk for result in mlt]), 22)
alt_mlt = self.sqs.filter(name='daniel3').more_like_this(
MockModel.objects.get(pk=13))
self.assertEqual(alt_mlt.count(), 8)
self.assertEqual(
sorted([result.pk for result in alt_mlt]),
sorted([u'4', u'3', u'22', u'19', u'17', u'16', u'10', u'23']))
self.assertEqual(len([result.pk for result in alt_mlt]), 8)
alt_mlt_with_models = self.sqs.models(MockModel).more_like_this(
MockModel.objects.get(pk=11))
self.assertEqual(alt_mlt_with_models.count(), 22)
self.assertEqual(
sorted([result.pk for result in alt_mlt_with_models]),
sorted([
u'9', u'8', u'7', u'6', u'5', u'4', u'3', u'2', u'1', u'22',
u'21', u'20', u'19', u'18', u'17', u'16', u'15', u'14', u'13',
u'12', u'10', u'23'
]))
self.assertEqual(len([result.pk for result in alt_mlt_with_models]),
22)
if hasattr(MockModel.objects, 'defer'):
# Make sure MLT works with deferred bits.
mi = MockModel.objects.defer('foo').get(pk=21)
self.assertEqual(mi._deferred, True)
deferred = self.sqs.models(MockModel).more_like_this(mi)
self.assertEqual(deferred.count(), 0)
self.assertEqual([result.pk for result in deferred], [])
self.assertEqual(len([result.pk for result in deferred]), 0)
# Ensure that swapping the ``result_class`` works.
self.assertTrue(
isinstance(
self.sqs.result_class(MockSearchResult).more_like_this(
MockModel.objects.get(pk=21))[0], MockSearchResult))
def results_for_members(self, team):
base_qs = SearchQuerySet().models(models.TeamVideo)
public = SQ(is_public=True)
mine = SQ(is_public=False, owned_by_team_id=team.pk)
return base_qs.filter(public | mine)
def build_haystack_filters(self, parameters):
from haystack.inputs import Raw
from haystack.query import SearchQuerySet, SQ # noqa
sqs = None
# Retrieve Query Params
# Text search
query = parameters.get('q', None)
# Types and subtypes to filter (map, layer, vector, etc)
type_facets = parameters.getlist("type__in", [])
# If coming from explore page, add type filter from resource_name
resource_filter = self._meta.resource_name.rstrip("s")
if resource_filter != "base" and resource_filter not in type_facets:
type_facets.append(resource_filter)
# Publication date range (start,end)
date_end = parameters.get("date__lte", None)
date_start = parameters.get("date__gte", None)
# Topic category filter
category = parameters.getlist("category__identifier__in")
# Keyword filter
keywords = parameters.getlist("keywords__slug__in")
# Region filter
regions = parameters.getlist("regions__name__in")
# Owner filters
owner = parameters.getlist("owner__username__in")
# Sort order
sort = parameters.get("order_by", "relevance")
# Geospatial Elements
bbox = parameters.get("extent", None)
# Filter by Type and subtype
if type_facets is not None:
types = []
subtypes = []
for type in type_facets:
if type in ["map", "layer", "document", "user"]:
# Type is one of our Major Types (not a sub type)
types.append(type)
elif type in LAYER_SUBTYPES.keys():
subtypes.append(type)
if len(subtypes) > 0:
types.append("layer")
sqs = SearchQuerySet().narrow("subtype:%s" %
','.join(map(str, subtypes)))
if len(types) > 0:
sqs = (SearchQuerySet() if sqs is None else sqs).narrow(
"type:%s" % ','.join(map(str, types)))
# Filter by Query Params
# haystack bug? if boosted fields aren't included in the
# query, then the score won't be affected by the boost
if query:
if query.startswith('"') or query.startswith('\''):
# Match exact phrase
phrase = query.replace('"', '')
sqs = (SearchQuerySet() if sqs is None else sqs).filter(
SQ(title__exact=phrase) |
SQ(description__exact=phrase) |
SQ(content__exact=phrase)
)
else:
words = [
w for w in re.split(
'\W',
query,
flags=re.UNICODE) if w]
for i, search_word in enumerate(words):
if i == 0:
sqs = (SearchQuerySet() if sqs is None else sqs) \
.filter(
SQ(title=Raw(search_word)) |
SQ(description=Raw(search_word)) |
SQ(content=Raw(search_word))
)
elif search_word in ["AND", "OR"]:
pass
elif words[i - 1] == "OR": # previous word OR this word
sqs = sqs.filter_or(
SQ(title=Raw(search_word)) |
SQ(description=Raw(search_word)) |
SQ(content=Raw(search_word))
)
else: # previous word AND this word
sqs = sqs.filter(
SQ(title=Raw(search_word)) |
SQ(description=Raw(search_word)) |
SQ(content=Raw(search_word))
)
# filter by category
if category:
sqs = (SearchQuerySet() if sqs is None else sqs).narrow(
'category:%s' % ','.join(map(str, category)))
# filter by keyword: use filter_or with keywords_exact
# not using exact leads to fuzzy matching and too many results
# using narrow with exact leads to zero results if multiple keywords
# selected
if keywords:
for keyword in keywords:
sqs = (
SearchQuerySet() if sqs is None else sqs).filter_or(
keywords_exact=keyword)
# filter by regions: use filter_or with regions_exact
# not using exact leads to fuzzy matching and too many results
# using narrow with exact leads to zero results if multiple keywords
# selected
if regions:
for region in regions:
sqs = (
SearchQuerySet() if sqs is None else sqs).filter_or(
regions_exact__exact=region)
# filter by owner
if owner:
sqs = (
SearchQuerySet() if sqs is None else sqs).narrow(
"owner__username:%s" % ','.join(map(str, owner)))
# filter by date
if date_start:
sqs = (SearchQuerySet() if sqs is None else sqs).filter(
SQ(date__gte=date_start)
)
if date_end:
sqs = (SearchQuerySet() if sqs is None else sqs).filter(
SQ(date__lte=date_end)
)
# Filter by geographic bounding box
if bbox:
left, bottom, right, top = bbox.split(',')
sqs = (
SearchQuerySet() if sqs is None else sqs).exclude(
SQ(
bbox_top__lte=bottom) | SQ(
bbox_bottom__gte=top) | SQ(
bbox_left__gte=right) | SQ(
bbox_right__lte=left))
# Apply sort
if sort.lower() == "-date":
sqs = (
SearchQuerySet() if sqs is None else sqs).order_by("-date")
elif sort.lower() == "date":
sqs = (
SearchQuerySet() if sqs is None else sqs).order_by("date")
elif sort.lower() == "title":
sqs = (SearchQuerySet() if sqs is None else sqs).order_by(
"title_sortable")
elif sort.lower() == "-title":
sqs = (SearchQuerySet() if sqs is None else sqs).order_by(
"-title_sortable")
elif sort.lower() == "-popular_count":
sqs = (SearchQuerySet() if sqs is None else sqs).order_by(
"-popular_count")
else:
sqs = (
SearchQuerySet() if sqs is None else sqs).order_by("-date")
return sqs
def do_site_search(q, allow_redirect=False, request=None):
if q.strip() == "":
return []
results = []
from bill.models import Bill
from vote.models import Vote
if "pass" in q or "fail" in q or "vote" in q:
results.append({
"title": "Tracking Federal Legislation",
"href": "/start",
"noun": "feeds",
"results": [
{"href": f.link,
"label": f.title,
"obj": f,
"feed": f,
"secondary": False }
for f in (
Bill.EnactedBillsFeed(), Bill.ActiveBillsExceptIntroductionsFeed(), Bill.ComingUpFeed(), Vote.AllVotesFeed(),
)
]
})
from haystack.query import SearchQuerySet
from events.models import Feed
from person.models import RoleType
sqs = SearchQuerySet().using("person")\
.filter(
indexed_model_name__in=["Person"],
all_role_types__in=(RoleType.representative, RoleType.senator),
content=q)
if 'XapianEngine' not in settings.HAYSTACK_CONNECTIONS['person']['ENGINE']:
# Xapian doesn't provide a 'score' so we can't do this when debugging.
sqs = sqs.order_by('-is_currently_serving', '-score')
results.append({
"title": "Members of Congress",
"href": "/congress/members/all",
"qsarg": "name",
"noun": "Members of Congress",
"results": [
{"href": p.object.get_absolute_url(),
"label": p.object.name,
"obj": p.object,
"feed": p.object.get_feed(),
"secondary": p.object.get_current_role() == None }
for p in sqs[0:9]]
})
import us
results.append({
"title": "States",
"href": "/congress/members",
"noun": "states",
"results": sorted([{"href": "/congress/members/%s" % s, "label": us.statenames[s] }
for s in us.statenames
if us.statenames[s].lower().startswith(q.lower())
], key=lambda p : p["label"])})
# search committees -- name must contain all of the words in the
# search query (e.g. "rules committee" should match "committee on rules")
from committee.models import Committee
committees_qs = Committee.objects.filter(obsolete=False)
for word in q.split(" "):
committees_qs = committees_qs.filter(name__icontains=word)
results.append({
"title": "Congressional Committees",
"href": "/congress/committees",
"noun": "committees in Congress",
"results": sorted([
{"href": c.get_absolute_url(),
"label": c.fullname,
"feed": c.get_feed(),
"obj": c,
"secondary": c.committee != None}
for c in committees_qs
], key=lambda c : c["label"])
})
from settings import CURRENT_CONGRESS
from bill.search import parse_bill_citation
bill = parse_bill_citation(q)
congress = "__ALL__"
if not bill or not allow_redirect:
# query Solr w/ the boosted field
from haystack.inputs import AutoQuery
from haystack.query import SQ
q = SearchQuerySet().using("bill").filter(indexed_model_name__in=["Bill"])\
.filter( SQ(text=AutoQuery(q)) | SQ(text_boosted=AutoQuery(q)) )
# restrict to current bills if any (at least 10) bills match
q1 = q.filter(congress=CURRENT_CONGRESS)
if q1.count() >= 10:
q = q1
congress = str(CURRENT_CONGRESS)
bills = [\
{"href": b.object.get_absolute_url(),
"label": b.object.title,
"obj": b.object,
"feed": b.object.get_feed() if b.object.is_alive else None,
"secondary": b.object.congress != CURRENT_CONGRESS }
for b in q[0:9]]
else:
url = bill.get_absolute_url()
if request.GET.get("track"): url += "#track"
return HttpResponseRedirect(url)
results.append({
"title": "Bills and Resolutions",
"href": "/congress/bills/browse",
"qsarg": "congress=%s&text" % congress,
"noun": "federal bills or resolutions",
"results": bills})
# subject terms, but exclude subject terms that look like committee names because
# that is confusing to also see with committee results
from bill.models import BillTerm, TermType
results.append({
"title": "Subject Areas",
"href": "/congress/bills",
"noun": "subject areas",
"results": [
{"href": p.get_absolute_url(),
"label": p.name,
"obj": p,
"feed": p.get_feed(),
"secondary": not p.is_top_term() }
for p in BillTerm.objects.filter(name__icontains=q, term_type=TermType.new).exclude(name__contains=" Committee on ")[0:9]]
})
# in each group, make sure the secondary results are placed last, but otherwise preserve order
for grp in results:
for i, obj in enumerate(grp["results"]):
obj["index"] = i
grp["results"].sort(key = lambda o : (o.get("secondary", False), o["index"]))
# sort categories first by whether all results are secondary results, then by number of matches (fewest first, if greater than zero)
results.sort(key = lambda c : (
len([d for d in c["results"] if d.get("secondary", False) == False]) == 0,
len(c["results"]) == 0,
len(c["results"])))
return results
class LiveWhooshRamStorageTestCase(TestCase):
def setUp(self):
super(LiveWhooshRamStorageTestCase, self).setUp()
# Stow.
self.old_whoosh_storage = getattr(settings, 'HAYSTACK_WHOOSH_STORAGE',
'file')
settings.HAYSTACK_WHOOSH_STORAGE = 'ram'
self.site = SearchSite()
self.sb = SearchBackend(site=self.site)
self.wrtsi = WhooshRoundTripSearchIndex(MockModel, backend=self.sb)
self.site.register(MockModel, WhooshRoundTripSearchIndex)
# Stow.
import haystack
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_site = haystack.site
haystack.site = self.site
self.sb.setup()
self.raw_whoosh = self.sb.index
self.parser = QueryParser(self.sb.content_field_name,
schema=self.sb.schema)
self.sqs = SearchQuerySet(site=self.site)
# Wipe it clean.
self.sqs.query.backend.clear()
# Fake indexing.
mock = MockModel()
mock.id = 1
self.sb.update(self.wrtsi, [mock])
def tearDown(self):
self.sqs.query.backend.clear()
settings.HAYSTACK_WHOOSH_STORAGE = self.old_whoosh_storage
import haystack
haystack.site = self.old_site
settings.DEBUG = self.old_debug
super(LiveWhooshRamStorageTestCase, self).tearDown()
def test_ram_storage(self):
results = self.sqs.filter(id='core.mockmodel.1')
# Sanity check.
self.assertEqual(results.count(), 1)
# Check the individual fields.
result = results[0]
self.assertEqual(result.id, 'core.mockmodel.1')
self.assertEqual(result.text, 'This is some example text.')
self.assertEqual(result.name, 'Mister Pants')
self.assertEqual(result.is_active, True)
self.assertEqual(result.post_count, 25)
self.assertEqual(result.average_rating, 3.6)
self.assertEqual(result.pub_date, datetime(2009, 11, 21, 0, 0))
self.assertEqual(result.created, datetime(2009, 11, 21, 21, 31, 00))
self.assertEqual(result.tags,
['staff', 'outdoor', 'activist', 'scientist'])
self.assertEqual(result.sites, [u'3', u'5', u'1'])
self.assertEqual(result.empty_list, [])
class LiveWhooshSearchQuerySetTestCase(TestCase):
def setUp(self):
super(LiveWhooshSearchQuerySetTestCase, self).setUp()
# Stow.
temp_path = os.path.join('tmp', 'test_whoosh_query')
self.old_whoosh_path = settings.HAYSTACK_CONNECTIONS['whoosh']['PATH']
settings.HAYSTACK_CONNECTIONS['whoosh']['PATH'] = temp_path
self.old_ui = connections['whoosh'].get_unified_index()
self.ui = UnifiedIndex()
self.wmmi = WhooshMockSearchIndex()
self.ui.build(indexes=[self.wmmi])
self.sb = connections['whoosh'].get_backend()
connections['whoosh']._index = self.ui
self.sb.setup()
self.raw_whoosh = self.sb.index
self.parser = QueryParser(self.sb.content_field_name,
schema=self.sb.schema)
self.sb.delete_index()
self.sample_objs = []
for i in range(1, 4):
mock = MockModel()
mock.id = i
mock.author = 'daniel%s' % i
mock.pub_date = date(2009, 2, 25) - timedelta(days=i)
self.sample_objs.append(mock)
self.sq = connections['whoosh'].get_query()
self.sqs = SearchQuerySet('whoosh')
def tearDown(self):
if os.path.exists(settings.HAYSTACK_CONNECTIONS['whoosh']['PATH']):
shutil.rmtree(settings.HAYSTACK_CONNECTIONS['whoosh']['PATH'])
settings.HAYSTACK_CONNECTIONS['whoosh']['PATH'] = self.old_whoosh_path
connections['whoosh']._index = self.old_ui
super(LiveWhooshSearchQuerySetTestCase, self).tearDown()
def test_various_searchquerysets(self):
self.sb.update(self.wmmi, self.sample_objs)
sqs = self.sqs.filter(content='Index')
self.assertEqual(sqs.query.build_query(), u'(Index)')
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!')
self.assertEqual(sqs.query.build_query(), u"('Indexed!')")
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!').filter(
pub_date__lte=date(2009, 8, 31))
self.assertEqual(sqs.query.build_query(),
u"(('Indexed!') AND pub_date:([to 20090831000000]))")
self.assertEqual(len(sqs), 3)
sqs = self.sqs.auto_query('Indexed!').filter(
pub_date__lte=date(2009, 2, 23))
self.assertEqual(sqs.query.build_query(),
u"(('Indexed!') AND pub_date:([to 20090223000000]))")
self.assertEqual(len(sqs), 2)
sqs = self.sqs.auto_query('Indexed!').filter(
pub_date__lte=date(2009, 2, 25)).filter(
django_id__in=[1, 2]).exclude(name='daniel1')
self.assertEqual(
sqs.query.build_query(),
u'((\'Indexed!\') AND pub_date:([to 20090225000000]) AND django_id:(1 OR 2) AND NOT (name:(daniel1)))'
)
self.assertEqual(len(sqs), 1)
sqs = self.sqs.auto_query('re-inker')
self.assertEqual(sqs.query.build_query(), u"('re-inker')")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.auto_query('0.7 wire')
self.assertEqual(sqs.query.build_query(), u"('0.7' wire)")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.auto_query("daler-rowney pearlescent 'bell bronze'")
self.assertEqual(sqs.query.build_query(),
u"('daler-rowney' pearlescent 'bell bronze')")
self.assertEqual(len(sqs), 0)
sqs = self.sqs.models(MockModel)
self.assertEqual(sqs.query.build_query(), u'*')
self.assertEqual(len(sqs), 3)
def test_all_regression(self):
sqs = SearchQuerySet('whoosh')
self.assertEqual([result.pk for result in sqs], [])
self.sb.update(self.wmmi, self.sample_objs)
self.assertTrue(self.sb.index.doc_count() > 0)
sqs = SearchQuerySet('whoosh')
self.assertEqual(len(sqs), 3)
self.assertEqual(sorted([result.pk for result in sqs]),
[u'1', u'2', u'3'])
try:
sqs = repr(SearchQuerySet('whoosh'))
except:
self.fail()
def test_regression_space_query(self):
self.sb.update(self.wmmi, self.sample_objs)
self.assertTrue(self.sb.index.doc_count() > 0)
sqs = SearchQuerySet('whoosh').auto_query(" ")
self.assertEqual(len(sqs), 3)
sqs = SearchQuerySet('whoosh').filter(content=" ")
self.assertEqual(len(sqs), 0)
def test_iter(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
sqs = self.sqs.auto_query('Indexed!')
results = [int(result.pk) for result in sqs]
self.assertEqual(sorted(results), [1, 2, 3])
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_slice(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(sorted([int(result.pk) for result in results[1:3]]),
[1, 2])
self.assertEqual(len(connections['whoosh'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(int(results[0].pk), 1)
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_manual_iter(self):
self.sb.update(self.wmmi, self.sample_objs)
results = self.sqs.auto_query('Indexed!')
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(sorted(results), [1, 2, 3])
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_fill_cache(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.auto_query('Indexed!')
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['whoosh'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 3)
self.assertEqual(len(connections['whoosh'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 3)
self.assertEqual(len(connections['whoosh'].queries), 2)
def test_cache_is_full(self):
self.sb.update(self.wmmi, self.sample_objs)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.auto_query('Indexed!')
[result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_count(self):
more_samples = []
for i in range(1, 50):
mock = MockModel()
mock.id = i
mock.author = 'daniel%s' % i
mock.pub_date = date(2009, 2, 25) - timedelta(days=i)
more_samples.append(mock)
self.sb.update(self.wmmi, more_samples)
reset_search_queries()
self.assertEqual(len(connections['whoosh'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results), 49)
self.assertEqual(results._cache_is_full(), False)
self.assertEqual(len(connections['whoosh'].queries), 1)
def test_query_generation(self):
sqs = self.sqs.filter(
SQ(content=AutoQuery("hello world"))
| SQ(title=AutoQuery("hello world")))
self.assertEqual(sqs.query.build_query(),
u"((hello world) OR title:(hello world))")
def test_result_class(self):
self.sb.update(self.wmmi, self.sample_objs)
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
