def get_global_context(self, context):
# Find all the models to search over...
models = set(self.search_sections[section]["model"] for section in self.search_sections)
show_top_hits = self.page == "1" or not self.page
top_hits_ids = []
if show_top_hits:
context["top_hits"] = []
for section, max_for_top_hits in SearchBaseView.top_hits_under.items():
data = self.get_section_data(section)
if data["results_count"] <= max_for_top_hits:
context["top_hits"] += data["results"]
top_hits_ids = set(r.id for r in context["top_hits"])
sqs = SearchQuerySet().models(*list(models))
# Exclude anything that will already have been shown in the top hits:
for top_hit_id in top_hits_ids:
sqs = sqs.exclude(id=top_hit_id)
sqs = sqs.exclude(hidden=True).filter(content=AutoQuery(self.query)).highlight()
if self.start_date_range:
sqs = sqs.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
sqs = sqs.filter(start_date__lte=self.end_date_range)
if self.order == "date":
sqs = sqs.order_by("-start_date")
context["paginator"] = Paginator(sqs, self.results_per_page)
context["page_obj"] = self.get_paginated_results(context["paginator"])
return context
def city_auto_search(request):
text = request.GET.get('text', '').split(', ')[:-1]
search = request.GET.get('location', '')
sqs = SearchQuerySet().models(City).filter(city_name__contains=search)
if text:
sqs = sqs.exclude(city_name__in=text)
suggestions = [{
'name': result.city_name,
'jobs_count': result.no_of_jobposts,
'id': result.pk
} for result in sqs]
suggestions = sorted(suggestions,
key=lambda k: len(k['name']),
reverse=False)
if not request.GET.get('search') == 'filter':
state = SearchQuerySet().models(State).filter_and(
SQ(state_name__contains=search) | SQ(state_slug__contains=search))
state = state.exclude(is_duplicate__in=[True])
if text:
state = state.exclude(state_name__in=text)
states = [{
'name': result.state_name,
'id': result.pk,
'slug': result.state_slug
} for result in state]
suggestions = suggestions + states
the_data = json.dumps({'results': suggestions[:10]})
return HttpResponse(the_data, content_type='application/json')
def get_object_list(self, request, **kwargs):
q = kwargs.get('query', '')
# sqs = SearchQuerySet().filter(SQ(content__contains=q) | SQ(name=q))
# sqs = SearchQuerySet().filter(SQ(content__contains=q) | SQ(content_auto=q))
#sqs = SearchQuerySet().filter(SQ(content=AutoQuery(q)) | SQ(name=AutoQuery(q)))
#sqs = SearchQuerySet().filter(content=AutoQuery(q))
sqs = SearchQuerySet().filter(text_auto=AutoQuery(q))
search_models = []
limit_models = request.GET.get('ct', None)
if limit_models:
for model in limit_models.split(':'):
try:
search_models.append(apps.get_model(*model.split('.')))
except LookupError as e:
pass
sqs = sqs.models(*search_models)
# TODO: nasty hack here. filter out 'basket' playlists that do not belong to the authenticated user
if request.user.is_authenticated():
sqs = sqs.exclude(Q(type='basket') | ~Q(user_pk=request.user.pk))
else:
sqs = sqs.exclude(type='basket')
sqs = sqs.highlight().load_all()
return sqs
def get_projects_ids_searched_by_title_tags_author(search_query, filter_or=False):
search_query_set = SearchQuerySet()
search_query_set = search_query_set.models(Project)
search_params = filter(None, search_query.split(' '))
# If no search params, then return all:
if not search_params:
return search_query_set.all()
sq_filter = None
for param in search_params:
if param.startswith('-'):
param = param[1:] # remove - sign
search_query_set = search_query_set.exclude(title=param)
search_query_set = search_query_set.exclude(tags=param)
search_query_set = search_query_set.exclude(owner_name=param)
else:
_sq_filter = SQ(title=param)
_sq_filter |= SQ(tags=param)
_sq_filter |= SQ(owner_name=param)
if sq_filter:
if filter_or:
sq_filter |= _sq_filter
else:
sq_filter &= _sq_filter
else:
sq_filter = _sq_filter
# Since sq.exclude.exclude.filter_or returns wrong data, we bypass it with a single .filter in the end:
if sq_filter is not None:
search_query_set = search_query_set.filter(sq_filter)
return search_query_set.values_list('pk', flat=True)
def skill_auto_search(request):
text = request.GET.get('text', '').split(', ')[:-1]
search = request.GET.get('q', '')
sqs = SearchQuerySet().models(Skill).filter_and(
SQ(skill_name__contains=search) | SQ(skill_slug__contains=search))
if text:
sqs = sqs.exclude(skill_name__in=text)
suggestions = [{
'name': result.skill_name,
'slug': result.skill_slug,
'jobs_count': result.no_of_jobposts,
'id': result.pk
} for result in sqs]
suggestions = sorted(suggestions,
key=lambda k: len(k['name']),
reverse=False)
if not request.GET.get('search') == 'filter':
deg = SearchQuerySet().models(Qualification).filter_and(
SQ(edu_name__contains=search) | SQ(edu_slug__contains=search))
if text:
deg = deg.exclude(edu_name__in=text)
degrees = [{
'name': result.edu_name,
'id': result.pk,
'slug': result.edu_slug
} for result in deg]
suggestions = suggestions + degrees
# suggestions = sorted(suggestions, key=int(itemgetter('jobs_count'), reverse=True)
the_data = json.dumps({'results': suggestions[:10]})
return HttpResponse(the_data, content_type='application/json')
def related_contact(request, selector='all'):
""" contact autocomplete
indexes: first_name, last_name, email, phone, site_user
type: 'all' (all), 'site' (site use only), 'no_site' (exclude site user)
"""
q = request.GET.get('q', None)
contact_id = request.GET.get('contact_id', None)
limit = 5
# Exclude already slaved contacts
exclude_idl = list(Contact.objects.all_filter(master_contact=contact_id).values_list('id', flat=True))
exclude_idl.append(contact_id)
""" query handler """
if q is None or q == '':
return HttpJsonResponse()
results = None
has_query = False
if not '@' in q:
query_list = process_query(q)
for q_o in query_list:
if q_o:
if not has_query:
has_query = True
results = SearchQuerySet().filter(SQ(first_name__startswith=q_o) | SQ(last_name__startswith=q_o) | \
SQ(email__startswith=q_o) | SQ(phone__startswith=q_o) | SQ(site_user__startswith=q_o))
else:
results = results.filter(SQ(first_name__startswith=q_o) | SQ(last_name__startswith=q_o) | \
SQ(email__startswith=q_o) | SQ(phone__startswith=q_o) | SQ(site_user__startswith=q_o))
else:
has_query = True
results = SearchQuerySet().auto_query(q)
if has_query:
results = results.filter(user=request.user.pk).exclude(is_slave=True).models(Contact).load_all()
else:
results = []
if has_query:
if selector == 'site':
results = results.filter(site_user__range=['*','*'])
elif selector == "no_site":
results = results.exclude(site_user__range=['*','*'])
results = results.exclude(cid__in=exclude_idl)
results = results.load_all()[:limit]
else:
results = []
tmp_response = []
html_content = ""
for result in results:
# Use html to render the contact
if not result:
continue
html_content += render_to_string('inc_contact_related.html', {'contact':result.object}, context_instance = RequestContext(request))
data = {'html':html_content}
return HttpJsonResponse(data)
def search(self):
def sorting_by_degree(result):
if result.model is Pages:
return '--'
try:
if result.object.in_followers(user):
return 1
v = result.object.get_degree_for(user)
except:
# this could happen
# run rebuild_index
v = '--'
return v
#res = super(SearchForm, self).search()
#res = SearchQuerySet().filter(username_auto=self.cleaned_data['q'])
if len(self.cleaned_data['q'].strip().split(" ")) > 1:
# if query has 2 and more words in it
# we can't use EdgeNgrams
res = SearchQuerySet().filter(content__contains=self.cleaned_data['q'])
else:
res = SearchQuerySet().filter(SQ(username_auto=self.cleaned_data['q']) | SQ(email=self.cleaned_data['q']) | SQ(full_name_auto=self.cleaned_data['q']))
# ajax filter
filter_val = self.data.get('filter')
# filtering
if filter_val == 'businesses':
res = res.filter(page_type='BS')
if filter_val == 'nonprofits':
res = res.filter(page_type='NP')
if filter_val == 'people':
res = res.models(UserProfile)
else:
res = res.models(UserProfile, Pages)
if 'account.userprofile' in self.cleaned_data['models']:
user = get_current_user()
for one_user in res:
#if hasattr(one_user.object, 'userprofile'):
if one_user.object and not isinstance(one_user.object,Tag) and not isinstance(one_user.object,Pages):
#if user was not deleted
if not one_user.object.check_visiblity("search",user):
res = res.exclude(username=one_user.username)
# if in blocked list -> exclude
if one_user.object in user.get_blocked():
res = res.exclude(username=one_user.username)
# Ensure the current user isn't in the results.
if user != None and one_user.object == user and not isinstance(one_user.object,Pages):
res = res.exclude(username=user.username)
# custom ordering (by DoS)
#res = sorted(res,key = lambda s : s.object.get_degree_for(user))
if filter_val in ('businesses','nonprofits'):
res = sorted(res,key = lambda s: s.object.loves, reverse=True)
else:
res = sorted(res,key = sorting_by_degree)
return res
def get_global_context(self, context):
# Find all the models to search over...
models = set(self.search_sections[section]['model']
for section in self.search_sections)
show_top_hits = (self.page == '1' or not self.page)
top_hits_ids = []
if show_top_hits:
context['top_hits'] = []
for section, max_for_top_hits in SearchBaseView.top_hits_under.items(
):
data = self.get_section_data(section)
if data['results_count'] <= max_for_top_hits:
context['top_hits'] += data['results']
top_hits_ids = set(r.id for r in context['top_hits'])
sqs = SearchQuerySet().models(*list(models))
# Exclude anything that will already have been shown in the top hits:
for top_hit_id in top_hits_ids:
sqs = sqs.exclude(id=top_hit_id)
sqs = sqs. \
exclude(hidden=True). \
filter(content=AutoQuery(self.query)). \
highlight()
context['results'] = self.get_paginated_results(sqs)
return context
def city_auto_search(request):
text = request.GET.get("text", "").split(", ")[:-1]
search = request.GET.get("location", "")
sqs = SearchQuerySet().models(City).filter(city_name__contains=search)
if text:
sqs = sqs.exclude(city_name__in=text)
suggestions = [{
"name": result.city_name,
"jobs_count": result.no_of_jobposts,
"id": result.pk
} for result in sqs]
suggestions = sorted(suggestions,
key=lambda k: len(k["name"]),
reverse=False)
if not request.GET.get("search") == "filter":
state = (SearchQuerySet().models(State).filter_and(
SQ(state_name__contains=search) | SQ(state_slug__contains=search)))
state = state.exclude(is_duplicate__in=[True])
if text:
state = state.exclude(state_name__in=text)
states = [{
"name": result.state_name,
"id": result.pk,
"slug": result.state_slug
} for result in state]
suggestions = suggestions + states
the_data = json.dumps({"results": suggestions[:10]})
return HttpResponse(the_data, content_type="application/json")
class LiveSimpleSearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSimpleSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SimpleMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sample_objs = MockModel.objects.all()
self.sqs = SearchQuerySet()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveSimpleSearchQuerySetTestCase, self).tearDown()
def test_general_queries(self):
# For now, just make sure these don't throw an exception.
# They won't work until the simple backend is improved.
self.assertTrue(len(self.sqs.auto_query('daniel')) > 0)
self.assertTrue(len(self.sqs.filter(text='index')) > 0)
self.assertTrue(len(self.sqs.exclude(name='daniel')) > 0)
self.assertTrue(len(self.sqs.order_by('-pub_date')) > 0)
class LiveSimpleSearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSimpleSearchQuerySetTestCase, self).setUp()
import haystack
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_site = haystack.site
self.site = SearchSite()
self.backend = SearchBackend(site=self.site)
self.index = SimpleMockSearchIndex(MockModel, backend=self.backend)
self.site.register(MockModel, SimpleMockSearchIndex)
haystack.site = self.site
self.sample_objs = MockModel.objects.all()
self.sqs = SearchQuerySet()
def tearDown(self):
# Restore.
import haystack
haystack.site = self.old_site
settings.DEBUG = self.old_debug
super(LiveSimpleSearchQuerySetTestCase, self).tearDown()
def test_general_queries(self):
# For now, just make sure these don't throw an exception.
# They won't work until the simple backend is improved.
self.assertTrue(len(self.sqs.auto_query('daniel')) > 0)
self.assertTrue(len(self.sqs.filter(text='index')) > 0)
self.assertTrue(len(self.sqs.exclude(name='daniel')) > 0)
self.assertTrue(len(self.sqs.order_by('-pub_date')) > 0)
def index(request):
profile = request.user.userprofile
queryset = Item.objects.filter(owner=request.user).order_by('-created').prefetch_related('tags')
sqs = SearchQuerySet().filter(owner_id=request.user.id)
if not profile.show_excluded:
excluded_tags = profile.excluded_tags.names()
queryset = queryset.without(*excluded_tags)
for tag in excluded_tags:
sqs = sqs.exclude(tags__in=[tag])
facets = sqs.facet('tags').facet_counts()
tag_objects = []
for name, count in facets.get('fields', {}).get('tags', []):
if name is not None:
tag_objects.append(Tag(name, count, []))
paginator = Paginator(queryset, profile.items_per_page)
try:
page = paginator.page(request.GET.get('page'))
except PageNotAnInteger:
page = paginator.page(1)
except EmptyPage:
page = paginator.page(paginator.num_pages)
context = {
'item_list': queryset,
'tags': tag_objects,
'tag_names': json.dumps([tag.name for tag in tag_objects]),
'current_item_list': page,
'user': request.user,
}
return TemplateResponse(request, 'readme/item_list.html', context)
def get_section_data(self, section):
defaults = self.search_sections[section]
extra_filter = defaults.get('filter', {})
filter_args = extra_filter.get('args', [])
filter_kwargs = extra_filter.get('kwargs', {})
extra_exclude = defaults.get('exclude', {})
query = SearchQuerySet().models(defaults['model'])
if extra_exclude:
query = query.exclude(**extra_exclude)
query = query.filter(content=self.generate_fuzzy_query_object(
self.query),
*filter_args,
**filter_kwargs)
if self.start_date_range:
query = query.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
query = query.filter(start_date__lte=self.end_date_range)
if self.order == 'date':
query = query.order_by('-start_date')
result = defaults.copy()
result['results'] = query.highlight()
result['results_count'] = result['results'].count()
result['section'] = section
result['section_dashes'] = section.replace('_', '-')
return result
def search(request):
sqs = SearchQuerySet().filter(content=Fuzzy(request.GET['q']))
if request.GET['id'] != '':
sqs = sqs.exclude(django_id=int(request.GET['id']))
res_list = sorted([x for x in sqs], key=lambda x: x.score, reverse=True)
ser = serializers.serialize('json', [x.object for x in res_list])
return HttpResponse(ser, content_type="application/json")
def tags(request, tags=''):
if tags == '':
return redirect(reverse('index'))
tag_list = [tag for tag in tags.split(',') if tag != '']
# Due to a bug (or feature?) in Whoosh or haystack, we can't filter for all tags at once,
# the .filter(tags=[...]) method cannot handle spaces apparently
# It however works with tags__in, but that is an OR
sqs = SearchQuerySet().filter(owner_id=request.user.id)
for tag in tag_list:
sqs = sqs.filter(tags__in=[tag])
profile = request.user.userprofile
if not profile.show_excluded:
excluded_tags = profile.excluded_tags.names()
for tag in excluded_tags:
sqs = sqs.exclude(tags__in=[tag])
sqs = sqs.order_by('-created').facet('tags')
facets = sqs.facet_counts()
result_objects = [result.object for result in sqs]
tag_objects = [Tag(name, count, tag_list) for name, count in facets.get('fields', {}).get('tags', [])]
return TemplateResponse(request, 'readme/item_list.html', {
'current_item_list': result_objects,
'tags': tag_objects,
'tag_names': json.dumps([tag.name for tag in tag_objects]),
'user': request.user,
})
class LiveSimpleSearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSimpleSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SimpleMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sample_objs = MockModel.objects.all()
self.sqs = SearchQuerySet()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveSimpleSearchQuerySetTestCase, self).tearDown()
def test_general_queries(self):
# For now, just make sure these don't throw an exception.
# They won't work until the simple backend is improved.
self.assertTrue(len(self.sqs.auto_query('daniel')) > 0)
self.assertTrue(len(self.sqs.filter(text='index')) > 0)
self.assertTrue(len(self.sqs.exclude(name='daniel')) > 0)
self.assertTrue(len(self.sqs.order_by('-pub_date')) > 0)
def get_global_context(self, context):
# Find all the models to search over...
models = set(
self.search_sections[section]['model']
for section in self.search_sections
)
show_top_hits = (self.page == '1' or not self.page)
top_hits_ids = []
if show_top_hits:
context['top_hits'] = []
for section, max_for_top_hits in SearchBaseView.top_hits_under.items():
data = self.get_section_data(section)
if data['results_count'] <= max_for_top_hits:
context['top_hits'] += data['results']
top_hits_ids = set(r.id for r in context['top_hits'])
sqs = SearchQuerySet().models(*list(models))
# Exclude anything that will already have been shown in the top hits:
for top_hit_id in top_hits_ids:
sqs = sqs.exclude(id=top_hit_id)
sqs = sqs. \
exclude(hidden=True). \
filter(content=AutoQuery(self.query)). \
highlight()
context['results'] = self.get_paginated_results(sqs)
return context
def get_projects_ids_searched_by_lesson_titles(search_query, filter_or=False):
search_query_set = SearchQuerySet()
search_query_set = search_query_set.models(Lesson)
search_params = filter(None, search_query.split(' '))
# If no search params, then return all:
if not search_params:
return set(search_query_set.values_list('project_id', flat=True))
sq_filter = None
for param in search_params:
if param.startswith('-'):
search_query_set = search_query_set.exclude(content=param[1:])
else:
_sq_filter = SQ(content__startswith=param)
if sq_filter:
if filter_or:
sq_filter |= _sq_filter
else:
sq_filter &= _sq_filter
else:
sq_filter = _sq_filter
# Since sq.exclude.exclude.filter_or returns wrong data, we bypass it with a single .filter in the end:
if sq_filter is not None:
search_query_set = search_query_set.filter(sq_filter)
return set(search_query_set.values_list('project_id', flat=True))
def get_section_data(self, section):
defaults = self.search_sections[section]
extra_filter = defaults.get('filter', {})
filter_args = extra_filter.get('args', [])
filter_kwargs = extra_filter.get('kwargs', {})
extra_exclude = defaults.get('exclude', {})
query = SearchQuerySet().models(defaults['model'])
if extra_exclude:
query = query.exclude(**extra_exclude)
query = query.filter(
content=AutoQuery(self.query),
*filter_args,
**filter_kwargs
)
if self.start_date_range:
query = query.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
query = query.filter(start_date__lte=self.end_date_range)
if self.order == 'date':
query = query.order_by('-start_date')
result = defaults.copy()
result['results'] = query.highlight()
result['results_count'] = result['results'].count()
result['section'] = section
result['section_dashes'] = section.replace('_', '-')
return result
def get_section_data(self, section):
defaults = self.search_sections[section]
extra_filter = defaults.get("filter", {})
filter_args = extra_filter.get("args", [])
filter_kwargs = extra_filter.get("kwargs", {})
extra_exclude = defaults.get("exclude", {})
query = SearchQuerySet().models(defaults["model"])
if extra_exclude:
query = query.exclude(**extra_exclude)
query = query.filter(content=AutoQuery(self.query), *filter_args, **filter_kwargs)
if self.start_date_range:
query = query.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
query = query.filter(start_date__lte=self.end_date_range)
if self.order == "date":
query = query.order_by("-start_date")
result = defaults.copy()
result["results"] = query.highlight()
result["results_count"] = result["results"].count()
result["section"] = section
result["section_dashes"] = section.replace("_", "-")
return result
def get_queryset(self):
qs = SearchQuerySet().all()
if self.request.user.is_authenticated():
# TODO: This has a leak! It should say:
# group=self.request.group.id AND group_read=True
if not permissions.can_moderate(models.URLPath.root().article,
self.request.user):
qs = qs.filter_or(owner=self.request.user.id,
group=self.request.group.id,
other_read=True)
else:
qs = qs.exclude(other_read=False)
qs = qs.filter(content=AutoQuery(self.query))
qs = qs.exclude(other_read=False)
qs = qs.load_all()
return qs
def get_queryset(self):
qs = SearchQuerySet().all()
if self.request.user.is_authenticated():
# TODO: This has a leak! It should say:
# group=self.request.group.id AND group_read=True
if not permissions.can_moderate(models.URLPath.root().article,
self.request.user):
qs = qs.filter_or(
owner=self.request.user.id,
group=self.request.group.id,
other_read=True
)
else:
qs = qs.exclude(other_read=False)
qs = qs.filter(content=AutoQuery(self.query))
qs = qs.exclude(other_read=False)
qs = qs.load_all()
return qs
def get_queryset(self):
queryset = super(LinkList, self).get_queryset().order_by('-added')
not_on_page = 'page' not in self.request.GET
if self.has_query():
queryset = SearchQuerySet().filter(content=AutoQuery(
self.request.GET['q']), ).filter_or(
categories=AutoQuery(self.request.GET['q'])).filter_or(
network_location=AutoQuery(self.request.GET['q']))
if self.has_favourites():
fav_pks = [l.pk for l in self.request.user.favourites.all()]
if self.has_query():
queryset = queryset.models(Link).filter(key__in=fav_pks)
else:
queryset = queryset.filter(id__in=fav_pks).distinct()
if self.has_categories():
categories_to_filter = dict(self.request.GET)['categories']
if type(categories_to_filter) == str:
categories_to_filter = [categories_to_filter]
if self.has_query():
queryset = queryset.models(Link).filter(
categories__in=categories_to_filter)
else:
# At this point, the queryset should already be ordered because
# of the original get_queryset call at the beginning of this
# function.
queryset = queryset.filter(
categories__name__in=categories_to_filter).distinct()
if self.external_only() or self.internal_only():
if self.has_query():
queryset = queryset.models(Link).exclude(
is_external=self.internal_only())
else:
# At this point, the queryset should already be ordered because
# of the original get_queryset call at the beginning of this
# function.
queryset = queryset.exclude(
is_external=self.internal_only()).distinct()
if self.has_query() and not self.has_categories() and not_on_page:
# At this point the queryset is a list of SearchResult objects, all
# of them. So, the length is accurate. By the time it reaches
# context, it won't be.
st, created = SearchTerm.objects.get_or_create(
query=self.request.GET.get('q'))
sq = SearchQuery()
sq.term = st
sq.results_length = len(queryset)
sq.user = self.request.user
sq.save()
return queryset
class LiveSimpleSearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSimpleSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SimpleMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sample_objs = MockModel.objects.all()
self.sqs = SearchQuerySet()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveSimpleSearchQuerySetTestCase, self).tearDown()
def test_general_queries(self):
# For now, just make sure these don't throw an exception.
# They won't work until the simple backend is improved.
self.assertTrue(len(self.sqs.auto_query('daniel')) > 0)
self.assertTrue(len(self.sqs.filter(text='index')) > 0)
self.assertTrue(len(self.sqs.exclude(name='daniel')) > 0)
self.assertTrue(len(self.sqs.order_by('-pub_date')) > 0)
def test_general_queries_unicode(self):
self.assertEqual(len(self.sqs.auto_query(u'Привет')), 0)
def test_more_like_this(self):
# MLT shouldn't be horribly broken. This used to throw an exception.
mm1 = MockModel.objects.get(pk=1)
self.assertEqual(len(self.sqs.filter(text=1).more_like_this(mm1)), 0)
def test_values_queries(self):
sqs = self.sqs.auto_query('daniel')
self.assertTrue(len(sqs) > 0)
flat_scores = sqs.values_list("score", flat=True)
self.assertEqual(flat_scores[0], 0)
scores = sqs.values_list("id", "score")
self.assertEqual(scores[0], [1, 0])
scores_dict = sqs.values("id", "score")
self.assertEqual(scores_dict[0], {"id": 1, "score": 0})
def ms_advanced_search(self, request, **kwargs):
get_params = request.GET.copy()
limit = get_params.get('limit', self._meta.limit)
sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags')
allWords = get_params.get('allWords', "").split(',')
exactExpressions = get_params.get('exactExpressions', "").split(',')
noneWord = get_params.get('noneWord', "").split(',')
allWords = [] if allWords == [''] else allWords
exactExpressions = [] if exactExpressions == [''] else exactExpressions
noneWord = [] if noneWord == [''] else noneWord
if get_params['searchIn'] in ['all', 'titles']:
for term in allWords:
sqs = sqs.filter_and(text=term.strip())
for term in exactExpressions:
sqs = sqs.filter_and(text=term.strip())
for term in noneWord:
sqs = sqs.exclude(text=term.strip())
elif get_params['searchIn'] == 'tags':
for term in allWords:
sqs = sqs.filter_and(tags=slugify(term.strip()))
for term in exactExpressions:
sqs = sqs.filter_and(tags=slugify(term.strip()))
for term in noneWord:
sqs = sqs.exclude(tags=slugify(term.strip()))
self.log_throttled_access(request)
return self.create_response(request,
self.prepare_result(request, sqs, limit))
def ms_advanced_search(self, request, **kwargs):
get_params = request.GET.copy()
limit = get_params.get('limit', self._meta.limit)
sqs = SearchQuerySet().models(self.Meta.object_class).facet('tags')
allWords = get_params.get('allWords', "").split(',')
exactExpressions = get_params.get('exactExpressions', "").split(',')
noneWord = get_params.get('noneWord', "").split(',')
allWords = [] if allWords == [''] else allWords
exactExpressions = [] if exactExpressions == [''] else exactExpressions
noneWord = [] if noneWord == [''] else noneWord
if get_params['searchIn'] in ['all', 'titles']:
for term in allWords:
sqs = sqs.filter_and(text=term.strip())
for term in exactExpressions:
sqs = sqs.filter_and(text=term.strip())
for term in noneWord:
sqs = sqs.exclude(text=term.strip())
elif get_params['searchIn'] == 'tags':
for term in allWords:
sqs = sqs.filter_and(tags=slugify(term.strip()))
for term in exactExpressions:
sqs = sqs.filter_and(tags=slugify(term.strip()))
for term in noneWord:
sqs = sqs.exclude(tags=slugify(term.strip()))
self.log_throttled_access(request)
return self.create_response(request, self.prepare_result(request, sqs, limit))
def get_queryset(self):
qs = SearchQuerySet().all()
if self.request.user.is_authenticated:
if not permissions.can_moderate(models.URLPath.root().article,
self.request.user):
qs = qs.filter(
SQ(owner_id=self.request.user.id)
| (SQ(group_id__in=self.request.user.groups.values_list(
'id', flat=True)) & SQ(group_read=True))
| SQ(other_read=True))
else:
qs = qs.exclude(other_read=False)
qs = qs.filter(content=AutoQuery(self.query))
qs = qs.load_all()
return qs
def read(self, request):
required_args(request, ['term'])
sqs = SearchQuerySet()
sqs = sqs.auto_query(request.GET['term']).load_all()
sqs = sqs.exclude(name__startswith="unknown")
results = {}
for i, result in enumerate(sqs):
if request.GET.get('format') == 'kml':
r = add_kml_to_recipient(result.object).__dict__
else:
r = result.object
results[i] = r
return results
def get_queryset(self):
qs = SearchQuerySet().all()
if self.request.user.is_authenticated():
if not permissions.can_moderate(models.URLPath.root().article, self.request.user):
qs = qs.filter(
SQ(owner_id=self.request.user.id) |
(
SQ(group_id__in=self.request.user.groups.values_list('id', flat=True))
& SQ(group_read=True)
) |
SQ(other_read=True)
)
else:
qs = qs.exclude(other_read=False)
qs = qs.filter(content=AutoQuery(self.query))
qs = qs.load_all()
return qs
def state_auto_search(request):
text = request.GET.get('text', '').split(', ')[:-1]
states = SearchQuerySet().models(State).filter_and(
SQ(state_name__contains=request.GET.get('state', ''))
| SQ(state_slug__contains=request.GET.get('state', '')))
if text:
states = states.exclude(state_name__in=text)
suggestions = [{
'name': result.state_name,
'id': result.pk,
'slug': result.state_slug,
'jobs_count': result.no_of_jobposts or 0
} for result in states]
suggestions = sorted(suggestions,
key=lambda k: int(k['jobs_count']),
reverse=True)
the_data = json.dumps({'results': suggestions[:10]})
return HttpResponse(the_data, content_type='application/json')
def filter_projects(request, default_limit=None, return_queryset=False):
host_slugs = parse_union(request)
contains = request.GET.get("search")
visibility = request.GET.get("visibility")
country_codes = parse_country(request)
limit = parse_limit(
request,
default=default_limit) if default_limit else parse_limit(request)
start, end = parse_filter_date(request)
results = SearchQuerySet()
if host_slugs:
results = results.filter(hosts_slug__in=host_slugs)
if visibility == 'completed':
results = results.filter_and(completed=True)
elif visibility == 'current':
results = results.exclude(completed=True)
if country_codes:
results = results.filter_and(country_code__in=country_codes)
if contains:
results = results.filter_and(text__contains=contains)
if start:
results &= SearchQuerySet().filter(
start_date__gte=start) | SearchQuerySet().filter(
end_date__gte=start)
if end:
results &= SearchQuerySet().filter(
start_date__lt=end) | SearchQuerySet().filter(end_date__lt=end)
results = results.models(Project).all()
if limit:
results = results[:limit]
if return_queryset:
projects_pk = [r.pk for r in results]
if len(host_slugs) == 1:
projects = Project.objects.filter(pk__in=projects_pk).order_by(
'completed', '-priority', '-updated')
else:
projects = Project.objects.filter(pk__in=projects_pk).order_by(
'completed', '-updated')
else:
projects = [result.object for result in results]
return projects
def autocomplete(request):
query = escape(request.GET.get('q', '')).strip()
language_code = get_language_from_request(request, check_path=True)
connection_alias = alias_from_language(language_code)
sqs = SearchQuerySet(using=connection_alias)
sqs = sqs.autocomplete(autocomplete=query)
sqs = sqs.filter(language=language_code)
if not request.user.is_authenticated():
sqs = sqs.exclude(login_required=True)
suggestions = []
for result in sqs[:8]:
suggestions.append({
# 'name': Highlighter(result.text).highlight(query),
'name': result.title,
'url': result.url,
})
return JSONResponse(request, suggestions)
def get_global_context(self, context):
# Find all the models to search over...
models = set(
self.search_sections[section]['model']
for section in self.search_sections
)
show_top_hits = (self.page == '1' or not self.page)
top_hits_ids = []
if show_top_hits:
context['top_hits'] = []
for section, max_for_top_hits in SearchBaseView.top_hits_under.items():
data = self.get_section_data(section)
if data['results_count'] <= max_for_top_hits:
context['top_hits'] += data['results']
top_hits_ids = set(r.id for r in context['top_hits'])
sqs = SearchQuerySet().models(*list(models))
# Exclude anything that will already have been shown in the top hits:
for top_hit_id in top_hits_ids:
sqs = sqs.exclude(id=top_hit_id)
sqs = sqs. \
exclude(hidden=True). \
filter(content=self.generate_fuzzy_query_object(self.query)). \
highlight()
if self.start_date_range:
sqs = sqs.filter(start_date__gte=self.start_date_range)
if self.end_date_range:
sqs = sqs.filter(start_date__lte=self.end_date_range)
if self.order == 'date':
sqs = sqs.order_by('-start_date')
context['paginator'] = Paginator(sqs, self.results_per_page)
context['page_obj'] = self.get_paginated_results(context['paginator'])
return context
def search(request, per_page=20):
"""
Search pages (and more in the future) in the current language based on keywords. Filtering based on published status
etc is already done at index level (that is, unpublished items are not included).
"""
query = escape(request.GET.get('q', '')).strip()
language_code = get_language_from_request(request, check_path=True)
language_name = dict(settings.LANGUAGES)[language_code]
if not query:
paginator = Paginator([], per_page)
page = paginator.page(1)
count = 0
else:
connection_alias = alias_from_language(language_code)
sqs = SearchQuerySet(using=connection_alias)
sqs = sqs.filter(text=AutoQuery(query))
sqs = sqs.filter(language=language_code)
if not request.user.is_authenticated():
sqs = sqs.exclude(login_required=True)
count = sqs.count()
paginator = Paginator(sqs, per_page)
try:
page = paginator.page(request.GET.get('page', 1))
except PageNotAnInteger:
page = paginator.page(1)
except EmptyPage:
page = Paginator(sqs, per_page)
return render_cms_special(request, 'search_content.html', dict(
query=query,
count=count,
results=page,
language_name=language_name,
))
class ItemSearchView(FacetedSearchView):
"""
SearchView that passes a dynamic SearchQuerySet to the
form that restricts the result to those owned by
the current user.
"""
def build_form(self, form_kwargs=None):
user_id = self.request.user.id
self.searchqueryset = SearchQuerySet().filter(owner_id=user_id).facet('tags')
profile = self.request.user.userprofile
if not profile.show_excluded:
for tag in profile.excluded_tags.names():
self.searchqueryset = self.searchqueryset.exclude(tags__in=[tag])
sorting = self.request.GET.get('sort', '')
if sorting == 'newest':
self.searchqueryset = self.searchqueryset.order_by('-created')
elif sorting == 'oldest':
self.searchqueryset = self.searchqueryset.order_by('created')
return super(ItemSearchView, self).build_form(form_kwargs)
def filter_partners(request, default_limit=None, return_queryset=False):
host_slugs = parse_union(request)
contains = request.GET.get("search")
active = request.GET.get("active")
categories = parse_categories(request)
limit = parse_limit(
request,
default=default_limit) if default_limit else parse_limit(request)
results = SearchQuerySet()
if host_slugs:
results = results.filter(hosts_slug__in=host_slugs)
if contains:
results = results.filter_and(text__contains=contains)
if categories:
results = results.filter_and(category__in=categories)
if active == 'active':
results = results.filter_and(active=True)
elif active == 'former':
results = results.exclude(active=True)
results = results.models(Partner).all()
if limit:
results = results[:limit]
if return_queryset:
partners_pk = [r.pk for r in results]
if len(host_slugs) == 1:
partners = Partner.objects.filter(pk__in=partners_pk).order_by(
'-active', '-priority', '-updated')
else:
partners = Partner.objects.filter(pk__in=partners_pk).order_by(
'-active', '-updated')
else: # for rest_views
partners = [result.object for result in results]
return partners
def get_potential_doubles(self):
"""
Returns an instance list of potential double for this race, based on :
- distance (<10km)
- date (+- 1 day)
- samed distance category
"""
sqs = SearchQuerySet()
pk = self.pk
if pk:
sqs = sqs.exclude(django_id=pk)
sqs = sqs.dwithin('location', self.location.get_point(), D(km=10))
sqs = sqs.filter(
date__gte=self.date + datetime.timedelta(days=-1),
date__lte=self.date + datetime.timedelta(days=1),
distance_cat=self.distance_cat.name,
sport__exact=self.sport.name,
# event_title=self.event.name
)
# return race objects instead of haystack searchresul
return [sr.object for sr in sqs]
def search(request):
# TODO: move to form
if 'action' in request.GET:
action = request.GET['action']
#FIXME: show_user for anonymous raise exception,
#django bug http://code.djangoproject.com/changeset/14087 :|
groups = request.user.groups.all() or [] #removed after django > 1.2.3 release
topics = Topic.objects.filter(
Q(forum__category__groups__in=groups) | \
Q(forum__category__groups__isnull=True))
if action == 'show_24h':
date = datetime.today() - timedelta(1)
topics = topics.filter(created__gte=date)
elif action == 'show_new':
try:
last_read = PostTracking.objects.get(user=request.user).last_read
except PostTracking.DoesNotExist:
last_read = None
if last_read:
topics = topics.filter(last_post__updated__gte=last_read).all()
else:
#searching more than forum_settings.SEARCH_PAGE_SIZE in this way - not good idea :]
topics = [topic for topic in topics[:forum_settings.SEARCH_PAGE_SIZE] if forum_extras.has_unreads(topic, request.user)]
elif action == 'show_unanswered':
topics = topics.filter(post_count=1)
elif action == 'show_subscriptions':
topics = topics.filter(subscribers__id=request.user.id)
elif action == 'show_user':
user_id = request.GET['user_id']
posts = Post.objects.filter(user__id=user_id)
topics = [post.topic for post in posts if post.topic in topics]
elif action == 'search':
keywords = request.GET.get('keywords')
author = request.GET.get('author')
forum = request.GET.get('forum')
search_in = request.GET.get('search_in')
sort_by = request.GET.get('sort_by')
sort_dir = request.GET.get('sort_dir')
if not (keywords or author):
return HttpResponseRedirect(reverse('djangobb:search'))
query = SearchQuerySet().models(Post)
if author:
query = query.filter(author__username=author)
if forum != u'0':
query = query.filter(forum__id=forum)
if keywords:
if search_in == 'all':
query = query.filter(SQ(topic=keywords) | SQ(text=keywords))
elif search_in == 'message':
query = query.filter(text=keywords)
elif search_in == 'topic':
query = query.filter(topic=keywords)
# add exlusions for categories user does not have access too
for category in Category.objects.all():
if not category.has_access(request.user):
query = query.exclude(category=category)
order = {'0': 'created',
'1': 'author',
'2': 'topic',
'3': 'forum'}.get(sort_by, 'created')
if sort_dir == 'DESC':
order = '-' + order
posts = query.order_by(order)
if 'topics' in request.GET['show_as']:
return render(request, 'djangobb_forum/search_topics.html', {
'results': TopicFromPostResult(posts)
})
elif 'posts' in request.GET['show_as']:
return render(request, 'djangobb_forum/search_posts.html', {'results': posts})
return render(request, 'djangobb_forum/search_topics.html', {'results': topics})
else:
form = PostSearchForm()
return render(request, 'djangobb_forum/search_form.html', {'categories': Category.objects.all(),
'form': form,
})
class SearchQuerySetTestCase(TestCase):
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
self.bsqs = SearchQuerySet(query=DummySearchQuery(backend=DummySearchBackend()))
self.msqs = SearchQuerySet(query=MockSearchQuery(backend=MockSearchBackend()))
self.mmsqs = SearchQuerySet(query=MockSearchQuery(backend=MixedMockSearchBackend()))
# Stow.
self.old_site = haystack.site
test_site = SearchSite()
test_site.register(MockModel)
haystack.site = test_site
def tearDown(self):
# Restore.
haystack.site = self.old_site
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
# Dummy always returns 0.
self.assertEqual(len(self.bsqs), 0)
self.assertEqual(len(self.msqs), 100)
def test_iter(self):
# Dummy always returns [].
self.assertEqual([result for result in self.bsqs.all()], [])
msqs = self.msqs.all()
results = [result for result in msqs]
self.assertEqual(results, MOCK_SEARCH_RESULTS)
def test_slice(self):
self.assertEqual(self.msqs.all()[1:11], MOCK_SEARCH_RESULTS[1:11])
self.assertEqual(self.msqs.all()[50], MOCK_SEARCH_RESULTS[50])
def test_manual_iter(self):
results = self.msqs.all()
for offset, result in enumerate(results._manual_iter()):
self.assertEqual(result, MOCK_SEARCH_RESULTS[offset])
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not in the SearchSite) than the hit count indicates.
# This will hang indefinitely if broken.
results = self.mmsqs.all()
self.assertEqual([result.pk for result in results._manual_iter()], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
def test_fill_cache(self):
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache()
self.assertEqual(len(results._result_cache), 10)
results._fill_cache()
self.assertEqual(len(results._result_cache), 20)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not in the SearchSite) than the hit count indicates.
results = self.mmsqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual([result.pk for result in results._result_cache], [])
results._fill_cache()
self.assertEqual(len(results._result_cache), 10)
self.assertEqual([result.pk for result in results._result_cache], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10])
results._fill_cache()
self.assertEqual(len(results._result_cache), 20)
self.assertEqual([result.pk for result in results._result_cache], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18, 19, 20, 21, 22])
results._fill_cache()
self.assertEqual(len(results._result_cache), 27)
self.assertEqual([result.pk for result in results._result_cache], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
def test_cache_is_full(self):
# Dummy always has a count of 0 and an empty _result_cache, hence True.
self.assertEqual(self.bsqs._cache_is_full(), True)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
def test_all(self):
sqs = self.bsqs.all()
self.assert_(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.bsqs.filter(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filters), 1)
def test_exclude(self):
sqs = self.bsqs.exclude(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filters), 1)
def test_order_by(self):
sqs = self.bsqs.order_by('foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assert_('foo' in sqs.query.order_by)
def test_models(self):
mock_index_site = SearchSite()
mock_index_site.register(MockModel)
mock_index_site.register(AnotherMockModel)
bsqs = SearchQuerySet(site=mock_index_site)
sqs = bsqs.all()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = bsqs.models(MockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = bsqs.models(MockModel, AnotherMockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
def test_boost(self):
sqs = self.bsqs.boost('foo', 10)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.bsqs.highlight()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.bsqs.filter(content='Indx')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
def test_raw_search(self):
self.assertEqual(len(self.bsqs.raw_search('foo')), 0)
self.assertEqual(len(self.bsqs.raw_search('content__exact hello AND content__exact world')), 1)
def test_load_all(self):
# If nothing is registered, you get nothing.
haystack.site.unregister(MockModel)
sqs = self.msqs.load_all()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
# For full tests, see the solr_backend.
def test_load_all_queryset(self):
sqs = self.msqs.load_all()
self.assertEqual(len(sqs._load_all_querysets), 0)
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
# For full tests, see the solr_backend.
def test_auto_query(self):
sqs = self.bsqs.auto_query('test search -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual([repr(the_filter) for the_filter in sqs.query.query_filters], ['<QueryFilter: AND content__exact=test>', '<QueryFilter: AND content__exact=search>', '<QueryFilter: NOT content__exact=stuff>'])
sqs = self.bsqs.auto_query('test "my thing" search -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual([repr(the_filter) for the_filter in sqs.query.query_filters], ['<QueryFilter: AND content__exact=my thing>', '<QueryFilter: AND content__exact=test>', '<QueryFilter: AND content__exact=search>', '<QueryFilter: NOT content__exact=stuff>'])
sqs = self.bsqs.auto_query('test "my thing" search \'moar quotes\' -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual([repr(the_filter) for the_filter in sqs.query.query_filters], ['<QueryFilter: AND content__exact=my thing>', '<QueryFilter: AND content__exact=moar quotes>', '<QueryFilter: AND content__exact=test>', '<QueryFilter: AND content__exact=search>', '<QueryFilter: NOT content__exact=stuff>'])
sqs = self.bsqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual([repr(the_filter) for the_filter in sqs.query.query_filters], ['<QueryFilter: AND content__exact=my thing>', '<QueryFilter: AND content__exact=moar quotes>', '<QueryFilter: AND content__exact=test>', '<QueryFilter: AND content__exact=search>', '<QueryFilter: AND content__exact="foo>', '<QueryFilter: NOT content__exact=stuff>'])
sqs = self.bsqs.auto_query('test - stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual([repr(the_filter) for the_filter in sqs.query.query_filters], ['<QueryFilter: AND content__exact=test>', '<QueryFilter: AND content__exact=->', '<QueryFilter: AND content__exact=stuff>'])
def test_count(self):
self.assertEqual(self.bsqs.count(), 0)
def test_facet_counts(self):
self.assertEqual(self.bsqs.facet_counts(), {})
def test_best_match(self):
self.assert_(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assert_(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 100)
def test_facets(self):
sqs = self.bsqs.facet('foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.bsqs.facet('foo').facet('bar')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
sqs = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month').date_facet('bar', start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='year')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.bsqs.query_facet('foo', '[bar TO *]')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.bsqs.query_facet('foo', '[bar TO *]').query_facet('bar', '[100 TO 499]')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
def test_narrow(self):
sqs = self.bsqs.narrow('foo:moof')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo='bar', foo__lt='10')
clone = results._clone()
self.assert_(isinstance(clone, SearchQuerySet))
self.assertEqual(clone.site, results.site)
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
def test_chaining(self):
sqs = self.bsqs.filter(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filters), 1)
# A second instance should inherit none of the changes from above.
sqs = self.bsqs.filter(content='bar')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filters), 1)
def test_none(self):
sqs = self.bsqs.none()
self.assert_(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def search(request):
# TODO: move to form
if 'action' in request.GET:
action = request.GET['action']
#FIXME: show_user for anonymous raise exception,
#django bug http://code.djangoproject.com/changeset/14087 :|
groups = request.user.groups.all() or [
] #removed after django > 1.2.3 release
topics = Topic.objects.filter(deleted=False).filter(
Q(forum__category__groups__in=groups) | \
Q(forum__category__groups__isnull=True))
if action == 'show_24h':
date = datetime.today() - timedelta(1)
topics = topics.filter(created__gte=date)
elif action == 'show_new':
try:
last_read = PostTracking.objects.get(
user=request.user).last_read
except PostTracking.DoesNotExist:
last_read = None
if last_read:
topics = topics.filter(last_post__updated__gte=last_read).all()
else:
#searching more than forum_settings.SEARCH_PAGE_SIZE in this way - not good idea :]
topics = [
topic for topic in topics[:forum_settings.SEARCH_PAGE_SIZE]
if forum_extras.has_unreads(topic, request.user)
]
elif action == 'show_unanswered':
topics = topics.filter(post_count=1)
elif action == 'show_subscriptions':
topics = topics.filter(subscribers__id=request.user.id)
elif action == 'show_user':
user_id = request.GET['user_id']
posts = Post.objects.filter(deleted=False, user__id=user_id)
topics = [post.topic for post in posts if post.topic in topics]
elif action == 'search':
keywords = request.GET.get('keywords')
author = request.GET.get('author')
forum = request.GET.get('forum')
search_in = request.GET.get('search_in')
sort_by = request.GET.get('sort_by')
sort_dir = request.GET.get('sort_dir')
if not (keywords or author):
return HttpResponseRedirect(reverse('djangobb:search'))
query = SearchQuerySet().models(Post).filter(deleted=0)
if author:
query = query.filter(author__username=author)
if forum != u'0':
query = query.filter(forum__id=forum)
if keywords:
if search_in == 'all':
query = query.filter(
SQ(topic=keywords) | SQ(text=keywords))
elif search_in == 'message':
query = query.filter(text=keywords)
elif search_in == 'topic':
query = query.filter(topic=keywords)
# add exlusions for categories user does not have access too
for category in Category.objects.all():
if not category.has_access(request.user):
query = query.exclude(category=category)
order = {
'0': 'created',
'1': 'author',
'2': 'topic',
'3': 'forum'
}.get(sort_by, 'created')
if sort_dir == 'DESC':
order = '-' + order
posts = query.order_by(order)
if 'topics' in request.GET['show_as']:
return render(request, 'djangobb_forum/search_topics.html',
{'results': TopicFromPostResult(posts)})
elif 'posts' in request.GET['show_as']:
return render(request, 'djangobb_forum/search_posts.html',
{'results': posts})
return render(request, 'djangobb_forum/search_topics.html',
{'results': topics})
else:
form = PostSearchForm()
return render(request, 'djangobb_forum/search_form.html', {
'categories': Category.objects.all(),
'form': form,
})
def map_search_endpoint(request, filter_group_id=None):
""" Maps API search endpoint using haystack search results. For parameters see ``MAP_SEARCH_PARAMETERS``
returns JSON with the contents of type ``HaystackMapResult``
@param filter_group_id: Will filter all items by group relation, where applicable
(i.e. users are filtered by group memberships for that group, events as events in that group)
"""
implicit_ignore_location = not any([
loc_param in request.GET
for loc_param in ['sw_lon', 'sw_lat', 'ne_lon', 'ne_lat']
])
params = _collect_parameters(request.GET, MAP_SEARCH_PARAMETERS)
query = force_text(params['q'])
limit = params['limit']
page = params['page']
item_id = params['item']
prefer_own_portal = getattr(settings, 'MAP_API_HACKS_PREFER_OWN_PORTAL',
False)
if not is_number(limit) or limit < 0:
return HttpResponseBadRequest(
'``limit`` param must be a positive number or 0!')
limit = min(limit, SERVER_SIDE_SEARCH_LIMIT)
if not is_number(page) or page < 0:
return HttpResponseBadRequest(
'``page`` param must be a positive number or 0!')
# filter for requested model types
model_list = [
klass for klass, param_name in list(SEARCH_MODEL_NAMES.items())
if params.get(param_name, False)
]
sqs = SearchQuerySet().models(*model_list)
# filter for map bounds (Points are constructed ith (lon, lat)!!!)
if not params['ignore_location'] and not implicit_ignore_location:
sqs = sqs.within('location', Point(params['sw_lon'], params['sw_lat']),
Point(params['ne_lon'], params['ne_lat']))
# filter for user's own content
if params['mine'] and request.user.is_authenticated:
user_id = request.user.id
sqs = sqs.filter_and(
Q(creator=user_id) | Q(user_id=user_id) | Q(group_members=user_id))
# filter for search terms
if query:
sqs = sqs.auto_query(query)
# group-filtered-map view for on-group pages
if filter_group_id:
group = get_object_or_None(get_cosinnus_group_model(),
id=filter_group_id)
if group:
filtered_groups = [filter_group_id]
# get child projects of this group
filtered_groups += [
subproject.id for subproject in group.get_children()
if subproject.is_active
]
sqs = sqs.filter_and(
Q(membership_groups__in=filtered_groups)
| Q(group__in=filtered_groups))
# filter topics
topics = ensure_list_of_ints(params.get('topics', ''))
if topics:
sqs = sqs.filter_and(mt_topics__in=topics)
# filter for portal visibility
sqs = filter_searchqueryset_for_portal(
sqs, restrict_multiportals_to_current=prefer_own_portal)
# filter for read access by this user
sqs = filter_searchqueryset_for_read_access(sqs, request.user)
# filter events by upcoming status
if params['events'] and Event is not None:
sqs = filter_event_searchqueryset_by_upcoming(sqs)
# filter all default user groups if the new dashboard is being used (they count as "on plattform" and aren't shown)
if getattr(settings, 'COSINNUS_USE_V2_DASHBOARD', False):
sqs = sqs.exclude(is_group_model=True,
slug__in=get_default_user_group_slugs())
# if we hae no query-boosted results, use *only* our custom sorting (haystack's is very random)
if not query:
if prefer_own_portal:
sqs = sqs.order_by('-portal', '-local_boost')
else:
sqs = sqs.order_by('-local_boost')
# sort results into one list per model
total_count = sqs.count()
sqs = sqs[limit * page:limit * (page + 1)]
results = []
for result in sqs:
# if we hae no query-boosted results, use *only* our custom sorting (haystack's is very random)
if not query:
result.score = result.local_boost
if prefer_own_portal and is_number(result.portal) and int(
result.portal) == CosinnusPortal.get_current().id:
result.score += 100.0
results.append(HaystackMapResult(result, user=request.user))
# if the requested item (direct select) is not in the queryset snippet
# (might happen because of an old URL), then mix it in as first item and drop the last
if item_id:
item_id = str(item_id)
if not any([res['id'] == item_id for res in results]):
item_result = get_searchresult_by_itemid(item_id, request.user)
if item_result:
results = [HaystackMapResult(item_result, user=request.user)
] + results[:-1]
page_obj = None
if results:
page_obj = {
'index': page,
'count': len(results),
'total_count': total_count,
'start': (limit * page) + 1,
'end': (limit * page) + len(results),
'has_next': total_count > (limit * (page + 1)),
'has_previous': page > 0,
}
data = {
'results': results,
'page': page_obj,
}
return JsonResponse(data)
def search(self):
def sorting_by_degree(result):
if result.model is Pages:
return '--'
try:
if result.object.in_followers(user):
return 1
v = result.object.get_degree_for(user)
except:
# this could happen
# run rebuild_index
v = '--'
return v
#res = super(SearchForm, self).search()
#res = SearchQuerySet().filter(username_auto=self.cleaned_data['q'])
if len(self.cleaned_data['q'].strip().split(" ")) > 1:
# if query has 2 and more words in it
# we can't use EdgeNgrams
res = SearchQuerySet().filter(
content__contains=self.cleaned_data['q'])
else:
res = SearchQuerySet().filter(
SQ(username_auto=self.cleaned_data['q'])
| SQ(email=self.cleaned_data['q'])
| SQ(full_name_auto=self.cleaned_data['q']))
# ajax filter
filter_val = self.data.get('filter')
# filtering
if filter_val == 'businesses':
res = res.filter(page_type='BS')
if filter_val == 'nonprofits':
res = res.filter(page_type='NP')
if filter_val == 'people':
res = res.models(UserProfile)
else:
res = res.models(UserProfile, Pages)
if 'account.userprofile' in self.cleaned_data['models']:
user = get_current_user()
for one_user in res:
#if hasattr(one_user.object, 'userprofile'):
if one_user.object and not isinstance(
one_user.object, Tag) and not isinstance(
one_user.object, Pages):
#if user was not deleted
if not one_user.object.check_visiblity("search", user):
res = res.exclude(username=one_user.username)
# if in blocked list -> exclude
if one_user.object in user.get_blocked():
res = res.exclude(username=one_user.username)
# Ensure the current user isn't in the results.
if user != None and one_user.object == user and not isinstance(
one_user.object, Pages):
res = res.exclude(username=user.username)
# custom ordering (by DoS)
#res = sorted(res,key = lambda s : s.object.get_degree_for(user))
if filter_val in ('businesses', 'nonprofits'):
res = sorted(res, key=lambda s: s.object.loves, reverse=True)
else:
res = sorted(res, key=sorting_by_degree)
return res
def get(self, request, *args, **kwargs):
"""
Primary endpoint for retrieving resources via the index
Values should never be empty string or python None, instead return string "None" with str() call
"availability": list value, js will parse JSON as Array
"availabilityurl": single value, pass a string to REST client
"type": single value, pass a string to REST client
"author": single value, pass a string to REST client first author
"creator: authors,
The reason for the weird name is the DataOne standard. The metadata was designed to be compliant
with DataOne standards. These standards do not contain an author field. Instead, the creator field
represents authors.
"contributor": list value, js will parse JSON as Array
"owner": list value, js will parse JSON as Array
"subject": list value, js will parse JSON as Array
"coverage_type": list point, period, ...
"""
start = time.time()
sqs = SearchQuerySet().all()
asc = '-1'
if request.GET.get('asc'):
asc = request.GET.get('asc')
sort = 'modified'
if request.GET.get('sort'):
sort = request.GET.get('sort')
sort = sort if asc == '1' else '-{}'.format(sort)
if request.GET.get('q'):
q = request.GET.get('q')
sqs = sqs.filter(content=q)
try:
qs = request.query_params
filters = json.loads(qs.get('filter'))
# filter values expect lists, for example discoverapi/?filter={"owner":["Firstname Lastname"]}
if filters.get('author'):
for k, authortype in enumerate(filters['author']):
if k == 0 or k == len(filters['author']):
sqs = sqs.filter(author_exact=Exact(authortype))
else:
sqs = sqs.filter_or(author_exact=Exact(authortype))
if filters.get('owner'):
for k, ownertype in enumerate(filters['owner']):
if k == 0 or k == len(filters['owner']):
sqs = sqs.filter(owner_exact=Exact(ownertype))
else:
sqs = sqs.filter_or(owner_exact=Exact(ownertype))
if filters.get('subject'):
for k, subjtype in enumerate(filters['subject']):
if k == 0 or k == len(subjtype):
sqs = sqs.filter(subject_exact=Exact(subjtype))
else:
sqs = sqs.filter_or(subject_exact=Exact(subjtype))
if filters.get('contributor'):
for k, contribtype in enumerate(filters['contributor']):
if k == 0 or k == len(filters['contributor']):
sqs = sqs.filter(contributor_exact=Exact(contribtype))
else:
sqs = sqs.filter_or(
contributor_exact=Exact(contribtype))
if filters.get('type'):
for k, restype in enumerate(filters['type']):
if k == 0 or k == len(filters['type']):
sqs = sqs.filter(content_type_exact=Exact(restype))
else:
sqs = sqs.filter_or(content_type_exact=Exact(restype))
if filters.get('availability'):
for k, availtype in enumerate(filters['availability']):
if k == 0 or k == len(filters['availability']):
sqs = sqs.filter(availability_exact=Exact(availtype))
else:
sqs = sqs.filter_or(
availability_exact=Exact(availtype))
if filters.get('geofilter'):
sqs = sqs.filter(
north__range=[-90,
90]) # return resources with geographic data
if filters.get('date'):
try:
datefilter = DateRange(start=datetime.datetime.strptime(
filters['date'][0], '%Y-%m-%d'),
end=datetime.datetime.strptime(
filters['date'][1], '%Y-%m-%d'))
# restrict to entries with dates
sqs = sqs.filter(start_date__gt=datetime.datetime.strptime('1900-01-01', '%Y-%m-%d'))\
.filter(end_date__lte=datetime.datetime.strptime(datetime.date.today().isoformat(), '%Y-%m-%d'))
# filter out entries that don't fall in specified range
sqs = sqs.exclude(start_date__gt=datefilter.end).exclude(
end_date__lt=datefilter.start)
except ValueError as date_ex:
return JsonResponse(
{
'message':
'Filter date parsing error expecting String %Y-%m-%d : {}'
.format(str(date_ex)),
'received':
request.query_params
},
status=400)
except Exception as gen_date_ex:
return JsonResponse(
{
'message':
'Filter date parsing error expecting two date string values : {}'
.format(str(gen_date_ex)),
'received':
request.query_params
},
status=400)
except TypeError as type_ex:
pass # no filters passed "the JSON object must be str, bytes or bytearray not NoneType"
except json.JSONDecodeError as parse_ex:
return JsonResponse(
{
'message':
'Filter JSON parsing error - {}'.format(str(parse_ex)),
'received':
request.query_params
},
status=400)
except Exception as gen_ex:
logger.warning('hs_discover API - {}: {}'.format(
type(gen_ex), str(gen_ex)))
return JsonResponse(
{
'message':
'{}'.format(
'{}: query error. Contact a server administrator.'.
format(type(gen_ex)))
},
status=520)
filterdata = []
if request.GET.get('filterbuilder'):
authors = sqs.facet('author').facet_counts()['fields']['author']
owners = sqs.facet('owner').facet_counts()['fields']['owner']
subjects = sqs.facet('subject').facet_counts()['fields']['subject']
contributors = sqs.facet(
'contributor').facet_counts()['fields']['contributor']
types = sqs.facet(
'content_type').facet_counts()['fields']['content_type']
availability = sqs.facet(
'availability').facet_counts()['fields']['availability']
if request.GET.get('updatefilters'):
authors = [x for x in authors if x[1] > 0]
owners = [x for x in owners if x[1] > 0]
subjects = [x for x in subjects if x[1] > 0]
contributors = [x for x in contributors if x[1] > 0]
types = [x for x in types if x[1] > 0]
availability = [x for x in availability if x[1] > 0]
filterdata = [
authors[:self.filterlimit], owners[:self.filterlimit],
subjects[:self.filterlimit], contributors[:self.filterlimit],
types[:self.filterlimit], availability[:self.filterlimit]
]
if sort == 'author':
sqs = sqs.order_by('author_exact')
elif sort == '-author':
sqs = sqs.order_by('-author_exact')
else:
sqs = sqs.order_by(sort)
resources = []
# TODO future release will add title and facilitate order_by title_exact
# convert sqs to list after facet operations to allow for Python sorting instead of Haystack order_by
if sort == 'title':
sqs = sorted(sqs, key=lambda idx: idx.title.lower())
elif sort == '-title':
sqs = sorted(sqs, key=lambda idx: idx.title.lower(), reverse=True)
p = Paginator(sqs, self.perpage)
if request.GET.get('pnum'):
pnum = request.GET.get('pnum')
pnum = int(pnum)
pnum = min(pnum, p.num_pages)
if pnum < 1:
return JsonResponse(
{
'resources': json.dumps([]),
'geodata': json.dumps([]),
'rescount': 0,
'pagecount': 1,
'perpage': self.perpage
},
status=200)
else:
pnum = 1 # page number not specified, implies page 1
pnum = min(pnum, p.num_pages)
geodata = []
for result in p.page(pnum):
contributor = 'None' # contributor is actually a list and can have multiple values
owner = 'None' # owner is actually a list and can have multiple values
author_link = None # Send None to avoid anchor render
creator = 'None'
author = 'None'
if result.creator:
creator = result.creator
authors = creator # there is no concept of authors in DataOne standard
# authors might be string 'None' here
if result.author:
author_link = result.author_url
author = str(result.author)
if authors == 'None':
authors = author # author would override creator in
else:
if result.organization:
if isinstance(result.organization, list):
author = str(result.organization[0])
else:
author = str(result.organization)
author = author.replace('"', '')
author = author.replace('[', '')
author = author.replace(']', '').strip()
if authors == 'None':
authors = author
if result.contributor is not None:
try:
contributor = result.contributor
except:
pass
if result.owner is not None:
try:
owner = result.owner
except:
pass
pt = '' # pass empty string for the frontend to ensure the attribute exists but can be evaluated for empty
try:
if 'box' in result.coverage_type:
pt = {
'short_id': result.short_id,
'title': result.title,
'coverage_type': 'box'
}
elif 'point' in result.coverage_type:
pt = {
'short_id': result.short_id,
'title': result.title,
'coverage_type': 'point'
}
if isinstance(result.north, (int, float)):
pt['north'] = result.north
if isinstance(result.east, (int, float)):
pt['east'] = result.east
if isinstance(result.northlimit, (int, float)):
pt['northlimit'] = result.northlimit
if isinstance(result.southlimit, (int, float)):
pt['southlimit'] = result.southlimit
if isinstance(result.eastlimit, (int, float)):
pt['eastlimit'] = result.eastlimit
if isinstance(result.westlimit, (int, float)):
pt['westlimit'] = result.westlimit
geodata.append(pt)
except:
pass # HydroShare production contains dirty data, this handling is in place, until data cleaned
resources.append({
"title":
result.title,
"link":
result.absolute_url,
"availability":
result.availability,
"availabilityurl":
"/static/img/{}.png".format(result.availability[0]),
"type":
result.resource_type_exact,
"author":
author,
"authors":
authors,
"contributor":
contributor,
"author_link":
author_link,
"owner":
owner,
"abstract":
result.abstract,
"subject":
result.subject,
"created":
result.created.isoformat(),
"modified":
result.modified.isoformat(),
"short_id":
result.short_id,
"geo":
pt
})
return JsonResponse(
{
'resources': json.dumps(resources),
'geodata': json.dumps(geodata),
'rescount': p.count,
'pagecount': p.num_pages,
'perpage': self.perpage,
'filterdata': json.dumps(filterdata),
'time': (time.time() - start) / 1000
},
status=200)
class SearchQuerySetTestCase(TestCase):
fixtures = ["base_data.json", "bulk_data.json"]
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections["default"]._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.uuidmmsi = SimpleMockUUIDModelIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi, self.uuidmmsi])
connections["default"]._index = self.ui
# Update the "index".
backend = connections["default"].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections["default"]._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object"
r" at 0x[0-9A-Fa-f]+>, using=None>$",
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in iter(msqs)]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections["default"].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(
[int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]],
)
self.assertEqual(len(connections["default"].queries), 1)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections["default"].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(
check,
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
],
)
self.assertEqual(len(connections["default"].queries), 3)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
# CharPK testing
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections["default"]._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, ["sometext", "1234"])
self.assertEqual(len(connections["default"].queries), 1)
# UUID testing
self.ui.build(indexes=[self.uuidmmsi])
connections["default"]._index = self.ui
self.uuidmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(
loaded,
[
"53554c58-7051-4350-bcc9-dad75eb248a9",
"77554c58-7051-4350-bcc9-dad75eb24888",
],
)
connections["default"]._index = old_ui
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = list(results)
self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections["default"].queries), 4)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue("foo" in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections["default"]._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections["default"]._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost("foo", 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling_override(self):
sqs = self.msqs.filter(content="not the spellchecking query")
self.assertEqual(sqs.query.spelling_query, None)
sqs = self.msqs.set_spelling_query("override")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.spelling_query, "override")
def test_spelling_suggestions(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content="Indx")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion("indexy"), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search("foo")), 23)
self.assertEqual(
len(
self.msqs.raw_search("(content__exact:hello AND content__exact:world)")
),
23,
)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend("charpk")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 2
)
# Models with uuid primary keys.
sqs = SearchQuerySet()
sqs.query.backend = UUIDMockSearchBackend("uuid")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 2
)
# If nothing is handled, you get nothing.
old_ui = connections["default"]._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections["default"]._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections["default"]._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections["default"]._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections["default"]._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 1
)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([result for result in results._result_cache if result is not None]), 2
)
# Restore.
connections["default"]._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query("test search -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND content__content=test search -stuff>",
)
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search -stuff>',
)
sqs = self.msqs.auto_query("test \"my thing\" search 'moar quotes' -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>",
)
sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>',
)
sqs = self.msqs.auto_query("test - stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), "<SQ: AND content__content=test - stuff>"
)
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), '<SQ: AND content__content="pants:rule">'
)
# Now with a different fieldname
sqs = self.msqs.auto_query("test search -stuff", fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), "<SQ: AND title__content=test search -stuff>"
)
sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND title__content=test "my thing" search -stuff>',
)
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet("foo").facet("bar")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="smarblaph",
)
self.fail()
except FacetingError as e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.",
)
sqs = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="month",
)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet(
"foo",
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="month",
).date_facet(
"bar",
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by="year",
)
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet("foo", "[bar TO *]")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet(
"bar", "[100 TO 499]"
)
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = (
self.msqs.query_facet("foo", "[bar TO *]")
.query_facet("bar", "[100 TO 499]")
.query_facet("foo", "[1000 TO 1499]")
)
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet("foo", "bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz")
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow("foo:moof")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo="bar", foo__lt="10")
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using="default")
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, "default")
def test_chaining(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content="bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter(content="oof")
sqs2 = self.msqs.filter(content="bar").filter(content="rab")
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, "OR")
self.assertEqual(
repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)
)
self.assertEqual(
repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter)
)
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter_or(content="oof")
sqs2 = self.msqs.filter(content="bar").filter_or(content="rab")
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, "AND")
self.assertEqual(
repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter)
)
self.assertEqual(
repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter)
)
def get_search(self, request, **kwargs):
# tests
self.method_check(request, allowed=['get'])
#self.is_authenticated(request)
self.throttle_check(request)
# pagination
limit = int(request.GET.get('limit', 100))
offset = int(request.GET.get('offset', 0))
page = (offset / limit) + 1
# Build query args
q_args = {'published': request.GET.get('published', True)}
narrow_args = []
# add search query
if 'q' in request.GET and request.GET['q'] != '':
q_args['content'] = AutoQuery(remove_accents(request.GET['q']))
# check for more params
params = ['category_id', 'category', 'user', 'user_id',
'status', 'id',
'created__year', 'created__month', 'created__day',
'country', 'admin_level1', 'admin_level2', 'admin_level3',
'has_images', 'is_geolocated', 'admin']
for p in params:
if p in request.GET:
q_args[self.rename_get_filters.get(p, p)] = request.GET.get(p)
# check for additional date filters (with datetime objects)
date_params = ['created__gt', 'created__lt', 'since', 'until']
for p in date_params:
if p in request.GET:
q_args[self.rename_get_filters.get(p, p)] = models.DateTimeField().to_python(request.GET.get(p))
if 'narrow_on_tags' in request.GET:
tags = map(normalize_tag, request.GET.get('narrow_on_tags').split(','))
narrow_args.append('tags:'+','.join(tags))
if 'tags' in request.GET:
tag_op = request.GET.get('tag_operator', 'or')
tags = map(normalize_tag, request.GET.get('tags').split(','))
if tag_op == 'or':
if len(tags) == 1 and tags[0].strip() != '':
q_args['tags__exact'] = tags[0]
else:
q_args['tags__exact__in'] = tags
elif tag_op == 'and':
narrow_args.append('tags:'+','.join(tags))
# GET ONLY DATEOS I FOLLOW INDIVIDUALLY
if 'followed' in request.GET:
uid = int(request.GET['followed'])
dateo_ids = [f.object_id for f in Follow.objects.filter(content_type__model='dateo', user__id=uid)]
q_args['id__in'] = dateo_ids
# GET DATEOS BY TAGS I FOLLOW
if 'followed_by_tags' in request.GET:
uid = int(request.GET['followed_by_tags'])
tags = [f.content_object.tag for f in Follow.objects.filter(content_type__model='tag', user__id=uid)]
q_args['tags__in'] = tags
# show published and unpublished actions
if q_args['published'] == 'all':
del q_args['published']
# INCLUDE REDATEOS (or not)
search_models = [Dateo]
if 'include_redateos' in request.GET and request.GET.get('include_redateos'):
search_models.append(Redateo)
# INIT THE QUERY
sqs = SearchQuerySet().models(*search_models).load_all()
for narg in narrow_args:
sqs = sqs.narrow(narg)
sqs = sqs.filter(**q_args)
# SPATIAL QUERY ADDONS
# WITHIN QUERY
if all(k in request.GET and request.GET.get(k) != '' for k in ('bottom_left_latitude',
'bottom_left_longitude','top_right_latitude', 'top_right_longitude')):
bl_x = float(request.GET.get('bottom_left_longitude'))
bl_y = float(request.GET.get('bottom_left_latitude'))
tr_x = float(request.GET.get('top_right_longitude'))
tr_y = float(request.GET.get('top_right_latitude'))
bottom_left = Point(bl_x, bl_y)
top_right = Point(tr_x, tr_y)
sqs = sqs.within('position', bottom_left, top_right)
# DWITHIN QUERY
if all(k in request.GET and request.GET.get(k) != '' for k in ('distance', 'latitude', 'longitude')):
dist = Distance( m = request.GET.get('distance'))
x = float(request.GET.get('longitude'))
y = float(request.GET.get('latitude'))
position = Point(x, y)
sqs = sqs.dwithin('position', position, dist)
# ADMIN CASE -> only new dateos
if 'new_in_campaign_id' in request.GET:
campaign_id = str(request.GET.get('new_in_campaign_id'))
sqs = sqs.exclude(admin__in=['reviewed:'+campaign_id, 'solved:'+campaign_id])
# ORDER BY
order_by = request.GET.get('order_by', '-created').split(',')
# if q is set, then order will be relevance first
# if not, then do normal order by
if 'q' in request.GET:
if order_by == ['-created'] and 'order_by' not in request.GET:
#order_by = ['_score']
order_by = ['score']
# if order_by distance, then things change a bit
if 'distance' in order_by and 'position' in request.GET and request.GET['position'] != '':
pos = [float(c) for c in request.GET.get('position').split(',')]
position = Point(pos[0], pos[1])
sqs = sqs.distance('position', position).order_by('distance', *order_by)
elif len(order_by) > 0:
sqs = sqs.order_by(*order_by)
paginator = Paginator(sqs, limit)
try:
page = paginator.page(page)
except InvalidPage:
raise Http404("Sorry, no results on that page.")
objects = []
for result in page.object_list:
res_obj = result.object if result.model_name == 'dateo' else result.object.dateo
cache_key = 'dateo.'+str(res_obj.pk)
data = self._meta.cache.get(cache_key)
if not data:
bundle = self.build_bundle(obj=res_obj, request=request)
bundle = self.full_dehydrate(bundle)
data = self._meta.cache.set(cache_key, bundle)
objects.append(data)
object_list = {
'meta': {
'limit': limit,
'next': page.has_next(),
'previous': page.has_previous(),
'total_count': sqs.count(),
'offset': offset
},
'objects': objects,
}
self.log_throttled_access(request)
return self.create_response(request, object_list)
class SearchQuerySetTestCase(TestCase):
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
self.bsqs = SearchQuerySet(query=DummySearchQuery(backend=DummySearchBackend()))
self.msqs = SearchQuerySet(query=MockSearchQuery(backend=MockSearchBackend()))
self.mmsqs = SearchQuerySet(query=MockSearchQuery(backend=MixedMockSearchBackend()))
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_site = haystack.site
test_site = SearchSite()
test_site.register(MockModel)
test_site.register(CharPKMockModel)
haystack.site = test_site
backends.reset_search_queries()
def tearDown(self):
# Restore.
haystack.site = self.old_site
settings.DEBUG = self.old_debug
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
# Dummy always returns 0.
self.assertEqual(len(self.bsqs), 0)
self.assertEqual(len(self.msqs), 100)
def test_repr(self):
self.assertEqual(repr(self.bsqs), '[]')
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
self.assertEqual(repr(self.msqs), "[<SearchResult: core.MockModel (pk=0)>, <SearchResult: core.MockModel (pk=1)>, <SearchResult: core.MockModel (pk=2)>, <SearchResult: core.MockModel (pk=3)>, <SearchResult: core.MockModel (pk=4)>, <SearchResult: core.MockModel (pk=5)>, <SearchResult: core.MockModel (pk=6)>, <SearchResult: core.MockModel (pk=7)>, <SearchResult: core.MockModel (pk=8)>, <SearchResult: core.MockModel (pk=9)>, <SearchResult: core.MockModel (pk=10)>, <SearchResult: core.MockModel (pk=11)>, <SearchResult: core.MockModel (pk=12)>, <SearchResult: core.MockModel (pk=13)>, <SearchResult: core.MockModel (pk=14)>, <SearchResult: core.MockModel (pk=15)>, <SearchResult: core.MockModel (pk=16)>, <SearchResult: core.MockModel (pk=17)>, <SearchResult: core.MockModel (pk=18)>, '...(remaining elements truncated)...']")
self.assertEqual(len(backends.queries), 1)
def test_iter(self):
# Dummy always returns [].
self.assertEqual([result for result in self.bsqs.all()], [])
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
msqs = self.msqs.all()
results = [result for result in msqs]
self.assertEqual(results, MOCK_SEARCH_RESULTS)
self.assertEqual(len(backends.queries), 10)
def test_slice(self):
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.msqs.all()
self.assertEqual(results[1:11], MOCK_SEARCH_RESULTS[1:11])
self.assertEqual(len(backends.queries), 1)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.msqs.all()
self.assertEqual(results[50], MOCK_SEARCH_RESULTS[50])
self.assertEqual(len(backends.queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
for offset, result in enumerate(results._manual_iter()):
self.assertEqual(result, MOCK_SEARCH_RESULTS[offset])
self.assertEqual(len(backends.queries), 10)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not in the SearchSite) than the hit count indicates.
# This will hang indefinitely if broken.
results = self.mmsqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
self.assertEqual(len(backends.queries), 8)
def test_fill_cache(self):
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(backends.queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(backends.queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(backends.queries), 2)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not in the SearchSite) than the hit count indicates.
results = self.mmsqs.all()
self.assertEqual(len([result for result in results._result_cache if result is not None]), 0)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [])
self.assertEqual(len(backends.queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 9)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [0, 1, 2, 3, 4, 5, 6, 7, 8])
self.assertEqual(len(backends.queries), 2)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 17)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19])
self.assertEqual(len(backends.queries), 4)
results._fill_cache(20, 30)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 27)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
self.assertEqual(len(backends.queries), 6)
def test_cache_is_full(self):
# Dummy always has a count of 0 and an empty _result_cache, hence True.
self.assertEqual(self.bsqs._cache_is_full(), False)
results = self.bsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(backends.queries), 10)
def test_all(self):
sqs = self.bsqs.all()
self.assert_(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.bsqs.filter(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.bsqs.exclude(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.bsqs.order_by('foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assert_('foo' in sqs.query.order_by)
def test_models(self):
mock_index_site = SearchSite()
mock_index_site.register(MockModel)
mock_index_site.register(AnotherMockModel)
bsqs = SearchQuerySet(site=mock_index_site)
sqs = bsqs.all()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = bsqs.models(MockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = bsqs.models(MockModel, AnotherMockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
mock_index_site.unregister(AnotherMockModel)
sqs = bsqs.models(AnotherMockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_boost(self):
sqs = self.bsqs.boost('foo', 10)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.bsqs.highlight()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.bsqs.filter(content='Indx')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.bsqs.raw_search('foo')), 0)
self.assertEqual(len(self.bsqs.raw_search('(content__exact hello AND content__exact world)')), 1)
def test_load_all(self):
# Models with character primary keys
sqs = SearchQuerySet(query=MockSearchQuery(backend=CharPKMockSearchBackend()))
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# If nothing is registered, you get nothing.
haystack.site.unregister(MockModel)
haystack.site.unregister(CharPKMockModel)
sqs = self.msqs.load_all()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
# For full tests, see the solr_backend.
def test_auto_query(self):
sqs = self.bsqs.auto_query('test search -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=test AND content__exact=search AND NOT (content__exact=stuff))>')
sqs = self.bsqs.auto_query('test "my thing" search -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=my thing AND content__exact=test AND content__exact=search AND NOT (content__exact=stuff))>')
sqs = self.bsqs.auto_query('test "my thing" search \'moar quotes\' -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND (content__exact=my thing AND content__exact=test AND content__exact=search AND content__exact='moar AND content__exact=quotes' AND NOT (content__exact=stuff))>")
sqs = self.bsqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=my thing AND content__exact=test AND content__exact=search AND content__exact=\'moar AND content__exact=quotes\' AND content__exact="foo AND NOT (content__exact=stuff))>')
sqs = self.bsqs.auto_query('test - stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=test AND content__exact=- AND content__exact=stuff)>')
# Ensure bits in exact matches get escaped properly as well.
sqs = self.bsqs.auto_query('"pants:rule"')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants:rule>')
def test_count(self):
self.assertEqual(self.bsqs.count(), 0)
def test_facet_counts(self):
self.assertEqual(self.bsqs.facet_counts(), {})
def test_best_match(self):
self.assert_(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assert_(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 100)
def test_facets(self):
sqs = self.bsqs.facet('foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.bsqs.facet('foo').facet('bar')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='smarblaph')
self.fail()
except FacetingError, e:
self.assertEqual(str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.")
sqs = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month').date_facet('bar', start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='year')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
class LiveSolrSearchQuerySetTestCase(TestCase):
"""Used to test actual implementation details of the SearchQuerySet."""
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSolrSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SolrMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
print 'Reloading data...'
lssqstc_all_loaded = True
# Wipe it clean.
clear_solr_index()
# Force indexing of the content.
self.smmi.update()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveSolrSearchQuerySetTestCase, self).tearDown()
def test_load_all(self):
sqs = self.sqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:")
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(int(results[21].pk), 22)
self.assertEqual(len(connections['default'].queries), 1)
def test_count(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
self.assertEqual(sqs.count(), 23)
self.assertEqual(sqs.count(), 23)
self.assertEqual(len(sqs), 23)
self.assertEqual(sqs.count(), 23)
# Should only execute one query to count the length of the result set.
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.sqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test___and__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')
# Now for something more complex...
sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar')
sqs = sqs3 & sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 3)
self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)')
def test___or__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')
# Now for something more complex...
sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar').models(MockModel)
sqs = sqs3 | sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)')
def test_auto_query(self):
# Ensure bits in exact matches get escaped properly as well.
# This will break horrifically if escaping isn't working.
sqs = self.sqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants\\:rule>')
self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"')
self.assertEqual(len(sqs), 0)
# Regressions
def test_regression_proper_start_offsets(self):
sqs = self.sqs.filter(text='index')
self.assertNotEqual(sqs.count(), 0)
id_counts = {}
for item in sqs:
if item.id in id_counts:
id_counts[item.id] += 1
else:
id_counts[item.id] = 1
for key, value in id_counts.items():
if value > 1:
self.fail("Result with id '%s' seen more than once in the results." % key)
def test_regression_raw_search_breaks_slicing(self):
sqs = self.sqs.raw_search('text: index')
page_1 = [result.pk for result in sqs[0:10]]
page_2 = [result.pk for result in sqs[10:20]]
for pk in page_2:
if pk in page_1:
self.fail("Result with id '%s' seen more than once in the results." % pk)
# RelatedSearchQuerySet Tests
def test_related_load_all(self):
sqs = self.rsqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(sqs[0].object.foo, u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:")
def test_related_load_all_queryset(self):
sqs = self.rsqs.load_all()
self.assertEqual(len(sqs._load_all_querysets), 0)
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], range(2, 24))
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])
def test_related_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.rsqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]], [2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(int(results[21].pk), 22)
self.assertEqual(len(connections['default'].queries), 4)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[20:30]], [21, 22, 23])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_manual_iter(self):
results = self.rsqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_related_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.rsqs._cache_is_full(), False)
results = self.rsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 5)
def test_quotes_regression(self):
sqs = self.sqs.auto_query("44°48'40''N 20°28'32''E")
# Should not have empty terms.
self.assertEqual(sqs.query.build_query(), u"(44\ufffd\ufffd48'40''N AND 20\ufffd\ufffd28'32''E)")
# Should not cause Solr to 500.
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing')
self.assertEqual(sqs.query.build_query(), u'blazing')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing saddles')
self.assertEqual(sqs.query.build_query(), u'(blazing AND saddles)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles')
self.assertEqual(sqs.query.build_query(), u'(\\"blazing AND saddles)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'"blazing saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'saddles"')
self.assertEqual(sqs.query.build_query(), u'("blazing \'saddles" AND mel)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel AND \')')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
self.assertEqual(sqs.query.build_query(), u'("blazing \'\'saddles" AND mel AND \'\\")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND brooks)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND brooks)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" AND mel AND \\"brooks)')
self.assertEqual(sqs.count(), 0)
def test_result_class(self):
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
class SearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections['default']._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections['default']._index = self.ui
# Update the "index".
backend = connections['default'].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections['default']._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r'^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object'
r' at 0x[0-9A-Fa-f]+>, using=None>$')
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in msqs]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(check, [
u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'10', u'11',
u'12', u'13', u'14', u'15', u'16', u'17', u'18', u'19', u'20',
u'21', u'22', u'23'
])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections['default']._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [u'sometext', u'1234'])
self.assertEqual(len(connections['default'].queries), 1)
connections['default']._index = old_ui
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
sqs = SearchQuerySet().all()
sqs.query.backend = MixedMockSearchBackend('default')
results = sqs
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 0)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [])
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 9)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10])
self.assertEqual(len(connections['default'].queries), 2)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 17)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20])
self.assertEqual(len(connections['default'].queries), 4)
results._fill_cache(20, 30)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [
1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22,
23
])
self.assertEqual(len(connections['default'].queries), 6)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue('foo' in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections['default']._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections['default']._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost('foo', 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content='Indx')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search('foo')), 23)
self.assertEqual(
len(
self.msqs.raw_search(
'(content__exact:hello AND content__exact:world)')), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend('charpk')
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# If nothing is handled, you get nothing.
old_ui = connections['default']._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections['default']._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections['default']._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections['default']._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections['default']._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# Restore.
connections['default']._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query('test search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains=test search -stuff>')
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search -stuff>')
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search \'moar quotes\' -stuff>'
)
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__contains=test "my thing" search \'moar quotes\' "foo -stuff>'
)
sqs = self.msqs.auto_query('test - stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND content__contains=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query('test search -stuff', fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
"<SQ: AND title__contains=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff',
fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND title__contains=test "my thing" search -stuff>')
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet('foo').facet('bar')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='smarblaph')
self.fail()
except FacetingError as e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second."
)
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month').date_facet(
'bar',
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='year')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet('foo', '[bar TO *]')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet('foo', '[bar TO *]').query_facet(
'bar', '[100 TO 499]')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = self.msqs.query_facet('foo', '[bar TO *]').query_facet(
'bar', '[100 TO 499]').query_facet('foo', '[1000 TO 1499]')
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet('foo', 'bar')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet('foo', 'bar').stats_facet('foo', 'baz')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet('foo', 'bar').stats_facet('moof', 'baz')
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow('foo:moof')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo='bar', foo__lt='10')
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using='default')
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, 'default')
def test_chaining(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content='bar')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content='foo')
sqs2 = self.msqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content='foo')
sqs2 = self.msqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content='foo').filter(content='oof')
sqs2 = self.msqs.filter(content='bar').filter(content='rab')
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, 'OR')
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content='foo').filter_or(content='oof')
sqs2 = self.msqs.filter(content='bar').filter_or(content='rab')
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, 'AND')
self.assertEqual(repr(sqs.query.query_filter.children[0]),
repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]),
repr(sqs2.query.query_filter))
def autocomplete(request):
"""Return autocomplete JSON results"""
term = request.GET.get('term','').strip()
response_data = []
if len(term):
# Does not work - probably because the FLAG_PARTIAL is not set on Xapian
# (trying to set it in settings.py as documented appears to have no effect)
# sqs = SearchQuerySet().autocomplete(name_auto=term)
# Split the search term up into little bits
terms = re.split(r'\s+', term)
# Build up a query based on the bits
sqs = SearchQuerySet()
for bit in terms:
# print "Adding '%s' to the '%s' query" % (bit,term)
sqs = sqs.filter_and(
name_auto__startswith = sqs.query.clean( bit )
)
sqs = sqs.exclude(hidden=False)
# If we have a kind then filter on that too
model_kind = request.GET.get('model', None)
if model_kind:
model = known_kinds.get(model_kind, None)
if model:
sqs = sqs.models(model)
else:
sqs = sqs.models(
models.Person,
models.Organisation,
models.Place,
models.PositionTitle,
)
# collate the results into json for the autocomplete js
for result in sqs.all()[0:10]:
object = result.object
css_class = object.css_class()
# use the specific field if it has one
if hasattr(object, 'name_autocomplete_html'):
label = object.name_autocomplete_html
else:
label = object.name
image_url = None
if hasattr(object, 'primary_image'):
image = object.primary_image()
if image:
image_url = get_thumbnail(image, '16x16', crop="center").url
if not image_url:
image_url = "/static/images/" + css_class + "-16x16.jpg"
response_data.append({
'url': object.get_absolute_url(),
'label': '<img height="16" width="16" src="%s" /> %s' % (image_url, label),
'type': css_class,
'value': object.name,
'object': object
})
remove_duplicate_places(response_data)
# Remove the 'object' elements before returning the response:
for d in response_data:
del d['object']
# send back the results as JSON
return HttpResponse(
simplejson.dumps(response_data),
content_type='application/json',
)
class SearchQuerySetTestCase(TestCase):
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
self.bsqs = SearchQuerySet(query=DummySearchQuery(backend=DummySearchBackend()))
self.msqs = SearchQuerySet(query=MockSearchQuery(backend=MockSearchBackend()))
self.mmsqs = SearchQuerySet(query=MockSearchQuery(backend=MixedMockSearchBackend()))
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_site = haystack.site
test_site = SearchSite()
test_site.register(MockModel)
test_site.register(CharPKMockModel)
haystack.site = test_site
backends.reset_search_queries()
def tearDown(self):
# Restore.
haystack.site = self.old_site
settings.DEBUG = self.old_debug
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
# Dummy always returns 0.
self.assertEqual(len(self.bsqs), 0)
self.assertEqual(len(self.msqs), 100)
def test_repr(self):
self.assertEqual(repr(self.bsqs), '[]')
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
self.assertEqual(repr(self.msqs), "[<SearchResult: core.MockModel (pk=0)>, <SearchResult: core.MockModel (pk=1)>, <SearchResult: core.MockModel (pk=2)>, <SearchResult: core.MockModel (pk=3)>, <SearchResult: core.MockModel (pk=4)>, <SearchResult: core.MockModel (pk=5)>, <SearchResult: core.MockModel (pk=6)>, <SearchResult: core.MockModel (pk=7)>, <SearchResult: core.MockModel (pk=8)>, <SearchResult: core.MockModel (pk=9)>, <SearchResult: core.MockModel (pk=10)>, <SearchResult: core.MockModel (pk=11)>, <SearchResult: core.MockModel (pk=12)>, <SearchResult: core.MockModel (pk=13)>, <SearchResult: core.MockModel (pk=14)>, <SearchResult: core.MockModel (pk=15)>, <SearchResult: core.MockModel (pk=16)>, <SearchResult: core.MockModel (pk=17)>, <SearchResult: core.MockModel (pk=18)>, '...(remaining elements truncated)...']")
self.assertEqual(len(backends.queries), 1)
def test_iter(self):
# Dummy always returns [].
self.assertEqual([result for result in self.bsqs.all()], [])
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
msqs = self.msqs.all()
results = [result for result in msqs]
self.assertEqual(results, MOCK_SEARCH_RESULTS)
self.assertEqual(len(backends.queries), 10)
def test_slice(self):
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.msqs.all()
self.assertEqual(results[1:11], MOCK_SEARCH_RESULTS[1:11])
self.assertEqual(len(backends.queries), 1)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.msqs.all()
self.assertEqual(results[50], MOCK_SEARCH_RESULTS[50])
self.assertEqual(len(backends.queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
for offset, result in enumerate(results._manual_iter()):
self.assertEqual(result, MOCK_SEARCH_RESULTS[offset])
self.assertEqual(len(backends.queries), 10)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not in the SearchSite) than the hit count indicates.
# This will hang indefinitely if broken.
results = self.mmsqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
self.assertEqual(len(backends.queries), 8)
def test_fill_cache(self):
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(backends.queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(backends.queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(backends.queries), 2)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not in the SearchSite) than the hit count indicates.
results = self.mmsqs.all()
self.assertEqual(len([result for result in results._result_cache if result is not None]), 0)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [])
self.assertEqual(len(backends.queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 9)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [0, 1, 2, 3, 4, 5, 6, 7, 8])
self.assertEqual(len(backends.queries), 2)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 17)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19])
self.assertEqual(len(backends.queries), 4)
results._fill_cache(20, 30)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 27)
self.assertEqual([result.pk for result in results._result_cache if result is not None], [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29])
self.assertEqual(len(backends.queries), 6)
def test_cache_is_full(self):
# Dummy always has a count of 0 and an empty _result_cache, hence True.
self.assertEqual(self.bsqs._cache_is_full(), False)
results = self.bsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
backends.reset_search_queries()
self.assertEqual(len(backends.queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(backends.queries), 10)
def test_all(self):
sqs = self.bsqs.all()
self.assert_(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.bsqs.filter(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.bsqs.exclude(content='foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.bsqs.order_by('foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assert_('foo' in sqs.query.order_by)
def test_models(self):
mock_index_site = SearchSite()
mock_index_site.register(MockModel)
mock_index_site.register(AnotherMockModel)
bsqs = SearchQuerySet(site=mock_index_site)
sqs = bsqs.all()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = bsqs.models(MockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = bsqs.models(MockModel, AnotherMockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
mock_index_site.unregister(AnotherMockModel)
sqs = bsqs.models(AnotherMockModel)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.bsqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.bsqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.bsqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.bsqs.boost('foo', 10)
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.bsqs.highlight()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.bsqs.filter(content='Indx')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.bsqs.raw_search('foo')), 0)
self.assertEqual(len(self.bsqs.raw_search('(content__exact hello AND content__exact world)')), 1)
def test_load_all(self):
# Models with character primary keys
sqs = SearchQuerySet(query=MockSearchQuery(backend=CharPKMockSearchBackend()))
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# If nothing is registered, you get nothing.
haystack.site.unregister(MockModel)
haystack.site.unregister(CharPKMockModel)
sqs = self.msqs.load_all()
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
# For full tests, see the solr_backend.
def test_auto_query(self):
sqs = self.bsqs.auto_query('test search -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=test AND content__exact=search AND NOT (content__exact=stuff))>')
sqs = self.bsqs.auto_query('test "my thing" search -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=my thing AND content__exact=test AND content__exact=search AND NOT (content__exact=stuff))>')
sqs = self.bsqs.auto_query('test "my thing" search \'moar quotes\' -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND (content__exact=my thing AND content__exact=test AND content__exact=search AND content__exact='moar AND content__exact=quotes' AND NOT (content__exact=stuff))>")
sqs = self.bsqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=my thing AND content__exact=test AND content__exact=search AND content__exact=\'moar AND content__exact=quotes\' AND content__exact="foo AND NOT (content__exact=stuff))>')
sqs = self.bsqs.auto_query('test - stuff')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND (content__exact=test AND content__exact=- AND content__exact=stuff)>')
# Ensure bits in exact matches get escaped properly as well.
sqs = self.bsqs.auto_query('"pants:rule"')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__exact=pants:rule>')
def test_count(self):
self.assertEqual(self.bsqs.count(), 0)
def test_facet_counts(self):
self.assertEqual(self.bsqs.facet_counts(), {})
def test_best_match(self):
self.assert_(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assert_(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 100)
def test_facets(self):
sqs = self.bsqs.facet('foo')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.bsqs.facet('foo').facet('bar')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='smarblaph')
self.fail()
except FacetingError, e:
self.assertEqual(str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.")
sqs = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month')
self.assert_(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.bsqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month').date_facet('bar', start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='year')
self.assert_(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def autocomplete(request):
"""Return autocomplete JSON results"""
term = request.GET.get('term', '').strip()
response_data = []
if len(term):
# Does not work - probably because the FLAG_PARTIAL is not set on Xapian
# (trying to set it in settings.py as documented appears to have no effect)
# sqs = SearchQuerySet().autocomplete(name_auto=term)
# Split the search term up into little bits
terms = re.split(r'\s+', term)
# Build up a query based on the bits
sqs = SearchQuerySet()
for bit in terms:
# print "Adding '%s' to the '%s' query" % (bit,term)
sqs = sqs.filter_and(name_auto__startswith=sqs.query.clean(bit))
sqs = sqs.exclude(hidden=False)
# If we have a kind then filter on that too
model_kind = request.GET.get('model', None)
if model_kind:
model = known_kinds.get(model_kind, None)
if model:
sqs = sqs.models(model)
else:
sqs = sqs.models(
models.Person,
models.Organisation,
models.Place,
models.PositionTitle,
)
# collate the results into json for the autocomplete js
for result in sqs.all()[0:10]:
o = result.object
css_class = o.css_class()
extra_autocomplete_data = None
if hasattr(o, 'extra_autocomplete_data'):
extra_autocomplete_data = o.extra_autocomplete_data
image_url = None
if hasattr(o, 'primary_image'):
image = o.primary_image()
if image:
image_url = get_thumbnail(image, '16x16',
crop="center").url
if not image_url:
image_url = "/static/images/" + css_class + "-16x16.jpg"
response_data.append({
'url': o.get_absolute_url(),
'name': o.name,
'image_url': image_url,
'extra_data': extra_autocomplete_data,
'type': css_class,
'value': o.name,
'object': o
})
remove_duplicate_places(response_data)
# Remove the 'object' elements before returning the response:
for d in response_data:
del d['object']
# send back the results as JSON
return HttpResponse(
simplejson.dumps(response_data),
content_type='application/json',
)
class SearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections['default']._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections['default']._index = self.ui
# Update the "index".
backend = connections['default'].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
reset_search_queries()
def tearDown(self):
# Restore.
connections['default']._index = self.old_unified_index
settings.DEBUG = self.old_debug
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(repr(self.msqs), "[<SearchResult: core.mockmodel (pk=u'1')>, <SearchResult: core.mockmodel (pk=u'2')>, <SearchResult: core.mockmodel (pk=u'3')>, <SearchResult: core.mockmodel (pk=u'4')>, <SearchResult: core.mockmodel (pk=u'5')>, <SearchResult: core.mockmodel (pk=u'6')>, <SearchResult: core.mockmodel (pk=u'7')>, <SearchResult: core.mockmodel (pk=u'8')>, <SearchResult: core.mockmodel (pk=u'9')>, <SearchResult: core.mockmodel (pk=u'10')>, <SearchResult: core.mockmodel (pk=u'11')>, <SearchResult: core.mockmodel (pk=u'12')>, <SearchResult: core.mockmodel (pk=u'13')>, <SearchResult: core.mockmodel (pk=u'14')>, <SearchResult: core.mockmodel (pk=u'15')>, <SearchResult: core.mockmodel (pk=u'16')>, <SearchResult: core.mockmodel (pk=u'17')>, <SearchResult: core.mockmodel (pk=u'18')>, <SearchResult: core.mockmodel (pk=u'19')>, '...(remaining elements truncated)...']")
self.assertEqual(len(connections['default'].queries), 1)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in msqs]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]], [res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(check, [u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'10', u'11', u'12', u'13', u'14', u'15', u'16', u'17', u'18', u'19', u'20', u'21', u'22', u'23'])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections['default']._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [u'sometext', u'1234'])
self.assertEqual(len(connections['default'].queries), 1)
connections['default']._index = old_ui
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
sqs = SearchQuerySet().all()
sqs.query.backend = MixedMockSearchBackend('default')
results = sqs
self.assertEqual(len([result for result in results._result_cache if result is not None]), 0)
self.assertEqual([int(result.pk) for result in results._result_cache if result is not None], [])
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 9)
self.assertEqual([int(result.pk) for result in results._result_cache if result is not None], [1, 2, 3, 4, 5, 6, 7, 8, 10])
self.assertEqual(len(connections['default'].queries), 2)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 17)
self.assertEqual([int(result.pk) for result in results._result_cache if result is not None], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20])
self.assertEqual(len(connections['default'].queries), 4)
results._fill_cache(20, 30)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual([int(result.pk) for result in results._result_cache if result is not None], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22, 23])
self.assertEqual(len(connections['default'].queries), 6)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue('foo' in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections['default']._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections['default']._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost('foo', 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content='Indx')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search('foo')), 23)
self.assertEqual(len(self.msqs.raw_search('(content__exact:hello AND content__exact:world)')), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend('charpk')
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# If nothing is handled, you get nothing.
old_ui = connections['default']._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections['default']._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections['default']._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections['default']._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections['default']._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(len([result for result in results._result_cache if result is not None]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# Restore.
connections['default']._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query('test search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains=test search -stuff>')
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains=test "my thing" search -stuff>')
sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains=test "my thing" search \'moar quotes\' -stuff>')
sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains=test "my thing" search \'moar quotes\' "foo -stuff>')
sqs = self.msqs.auto_query('test - stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__contains=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query('test search -stuff', fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND title__contains=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND title__contains=test "my thing" search -stuff>')
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet('foo').facet('bar')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='smarblaph')
self.fail()
except FacetingError, e:
self.assertEqual(str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second.")
sqs = self.msqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet('foo', start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='month').date_facet('bar', start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by='year')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
class LiveElasticsearchSearchQuerySetTestCase(TestCase):
"""Used to test actual implementation details of the SearchQuerySet."""
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveElasticsearchSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = ElasticsearchMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
print 'Reloading data...'
lssqstc_all_loaded = True
# Wipe it clean.
clear_elasticsearch_index()
# Force indexing of the content.
self.smmi.update()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown()
def test_load_all(self):
sqs = self.sqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
results = sorted([int(result.pk) for result in sqs])
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(int(results[21].pk), 18)
self.assertEqual(len(connections['default'].queries), 1)
def test_count(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
self.assertEqual(sqs.count(), 23)
self.assertEqual(sqs.count(), 23)
self.assertEqual(len(sqs), 23)
self.assertEqual(sqs.count(), 23)
# Should only execute one query to count the length of the result set.
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.sqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
self.assertEqual(len(connections['default'].queries), 3)
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test___and__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'(foo AND bar)')
# Now for something more complex...
sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar')
sqs = sqs3 & sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 3)
self.assertEqual(sqs.query.build_query(), u'(NOT (title:moof) AND (foo OR baz) AND bar)')
def test___or__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'(foo OR bar)')
# Now for something more complex...
sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar').models(MockModel)
sqs = sqs3 | sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((NOT (title:moof) AND (foo OR baz)) OR bar)')
def test_auto_query(self):
# Ensure bits in exact matches get escaped properly as well.
# This will break horrifically if escaping isn't working.
sqs = self.sqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">')
self.assertEqual(sqs.query.build_query(), u'"pants\\:rule"')
self.assertEqual(len(sqs), 0)
# Regressions
def test_regression_proper_start_offsets(self):
sqs = self.sqs.filter(text='index')
self.assertNotEqual(sqs.count(), 0)
id_counts = {}
for item in sqs:
if item.id in id_counts:
id_counts[item.id] += 1
else:
id_counts[item.id] = 1
for key, value in id_counts.items():
if value > 1:
self.fail("Result with id '%s' seen more than once in the results." % key)
def test_regression_raw_search_breaks_slicing(self):
sqs = self.sqs.raw_search('text:index')
page_1 = [result.pk for result in sqs[0:10]]
page_2 = [result.pk for result in sqs[10:20]]
for pk in page_2:
if pk in page_1:
self.fail("Result with id '%s' seen more than once in the results." % pk)
# RelatedSearchQuerySet Tests
def test_related_load_all(self):
sqs = self.rsqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')
def test_related_load_all_queryset(self):
sqs = self.rsqs.load_all()
self.assertEqual(len(sqs._load_all_querysets), 0)
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual(sorted([obj.object.id for obj in sqs]), range(2, 24))
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], [12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20])
self.assertEqual([obj.object.id for obj in sqs[10:20]], [13, 18, 20])
def test_related_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.rsqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(int(results[21].pk), 18)
self.assertEqual(len(connections['default'].queries), 4)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[20:30]], [13, 18, 20])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_manual_iter(self):
results = self.rsqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = sorted([int(result.pk) for result in results._manual_iter()])
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_related_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.rsqs._cache_is_full(), False)
results = self.rsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 5)
def test_quotes_regression(self):
sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
# Should not have empty terms.
self.assertEqual(sqs.query.build_query(), u"44\xb048'40''N 20\xb028'32''E")
# Should not cause Elasticsearch to 500.
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing')
self.assertEqual(sqs.query.build_query(), u'blazing')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing saddles')
self.assertEqual(sqs.query.build_query(), u'blazing saddles')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles')
self.assertEqual(sqs.query.build_query(), u'\\"blazing saddles')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'"blazing saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'saddles"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing \'saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles" \'')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
self.assertEqual(sqs.query.build_query(), u'mel "blazing \'\'saddles" \'\\"')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel')
self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
self.assertEqual(sqs.query.build_query(), u'"blazing saddles" mel brooks')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles" brooks')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
self.assertEqual(sqs.query.build_query(), u'mel "blazing saddles" \\"brooks')
self.assertEqual(sqs.count(), 0)
def test_result_class(self):
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
class LiveElasticsearchSearchQuerySetTestCase(TestCase):
"""Used to test actual implementation details of the SearchQuerySet."""
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveElasticsearchSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = ElasticsearchMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
lssqstc_all_loaded = True
# Wipe it clean.
clear_elasticsearch_index()
# Force indexing of the content.
self.smmi.update()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveElasticsearchSearchQuerySetTestCase, self).tearDown()
def test_load_all(self):
sqs = self.sqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
results = sorted([int(result.pk) for result in sqs])
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(int(results[21].pk), 18)
self.assertEqual(len(connections['default'].queries), 1)
def test_count(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
self.assertEqual(sqs.count(), 23)
self.assertEqual(sqs.count(), 23)
self.assertEqual(len(sqs), 23)
self.assertEqual(sqs.count(), 23)
# Should only execute one query to count the length of the result set.
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.sqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
self.assertEqual(len(connections['default'].queries), 3)
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test___and__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))')
# Now for something more complex...
sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar')
sqs = sqs3 & sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 3)
self.assertEqual(sqs.query.build_query(), u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))')
def test___or__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))')
# Now for something more complex...
sqs3 = self.sqs.exclude(title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar').models(MockModel)
sqs = sqs3 | sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))')
def test_auto_query(self):
# Ensure bits in exact matches get escaped properly as well.
# This will break horrifically if escaping isn't working.
sqs = self.sqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__contains="pants:rule">')
self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")')
self.assertEqual(len(sqs), 0)
# Regressions
@unittest.expectedFailure
def test_regression_proper_start_offsets(self):
sqs = self.sqs.filter(text='index')
self.assertNotEqual(sqs.count(), 0)
id_counts = {}
for item in sqs:
if item.id in id_counts:
id_counts[item.id] += 1
else:
id_counts[item.id] = 1
for key, value in id_counts.items():
if value > 1:
self.fail("Result with id '%s' seen more than once in the results." % key)
def test_regression_raw_search_breaks_slicing(self):
sqs = self.sqs.raw_search('text:index')
page_1 = [result.pk for result in sqs[0:10]]
page_2 = [result.pk for result in sqs[10:20]]
for pk in page_2:
if pk in page_1:
self.fail("Result with id '%s' seen more than once in the results." % pk)
# RelatedSearchQuerySet Tests
def test_related_load_all(self):
sqs = self.rsqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(sqs[0].object.foo, u'In addition, you may specify other fields to be populated along with the document. In this case, we also index the user who authored the document as well as the date the document was published. The variable you assign the SearchField to should directly map to the field your search backend is expecting. You instantiate most search fields with a parameter that points to the attribute of the object to populate that field with.')
def test_related_load_all_queryset(self):
sqs = self.rsqs.load_all()
self.assertEqual(len(sqs._load_all_querysets), 0)
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=1))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual(sorted([obj.object.id for obj in sqs]), range(2, 24))
sqs = sqs.load_all_queryset(MockModel, MockModel.objects.filter(id__gt=10))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], [12, 17, 11, 16, 23, 15, 22, 14, 19, 21, 13, 18, 20])
self.assertEqual([obj.object.id for obj in sqs[10:20]], [13, 18, 20])
def test_related_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.rsqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, [2, 7, 12, 17, 1, 6, 11, 16, 23, 5, 10, 15, 22, 4, 9, 14, 19, 21, 3, 8, 13, 18, 20])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]], [7, 12, 17, 1, 6, 11, 16, 23, 5, 10])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(int(results[21].pk), 18)
self.assertEqual(len(connections['default'].queries), 4)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[20:30]], [13, 18, 20])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_manual_iter(self):
results = self.rsqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = sorted([int(result.pk) for result in results._manual_iter()])
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_related_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.rsqs._cache_is_full(), False)
results = self.rsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 5)
def test_quotes_regression(self):
sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
# Should not have empty terms.
self.assertEqual(sqs.query.build_query(), u"(44\xb048'40''N 20\xb028'32''E)")
# Should not cause Elasticsearch to 500.
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing')
self.assertEqual(sqs.query.build_query(), u'(blazing)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing saddles')
self.assertEqual(sqs.query.build_query(), u'(blazing saddles)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles')
self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'saddles"')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \')')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'\'saddles" \'\\")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel brooks)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" brooks)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles" \\"brooks)')
self.assertEqual(sqs.count(), 0)
def test_query_generation(self):
sqs = self.sqs.filter(SQ(content=AutoQuery("hello world")) | SQ(title=AutoQuery("hello world")))
self.assertEqual(sqs.query.build_query(), u"((hello world) OR title:(hello world))")
def test_result_class(self):
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
class LiveSolrSearchQuerySetTestCase(TestCase):
"""Used to test actual implementation details of the SearchQuerySet."""
fixtures = ['bulk_data.json']
def setUp(self):
super(LiveSolrSearchQuerySetTestCase, self).setUp()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
self.old_ui = connections['default'].get_unified_index()
self.ui = UnifiedIndex()
self.smmi = SolrMockSearchIndex()
self.ui.build(indexes=[self.smmi])
connections['default']._index = self.ui
self.sqs = SearchQuerySet()
self.rsqs = RelatedSearchQuerySet()
# Ugly but not constantly reindexing saves us almost 50% runtime.
global lssqstc_all_loaded
if lssqstc_all_loaded is None:
print 'Reloading data...'
lssqstc_all_loaded = True
# Wipe it clean.
clear_solr_index()
# Force indexing of the content.
self.smmi.update()
def tearDown(self):
# Restore.
connections['default']._index = self.old_ui
settings.DEBUG = self.old_debug
super(LiveSolrSearchQuerySetTestCase, self).tearDown()
def test_load_all(self):
sqs = self.sqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(
sqs[0].object.foo,
u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]],
[2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(int(results[21].pk), 22)
self.assertEqual(len(connections['default'].queries), 1)
def test_count(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.sqs.all()
self.assertEqual(sqs.count(), 23)
self.assertEqual(sqs.count(), 23)
self.assertEqual(len(sqs), 23)
self.assertEqual(sqs.count(), 23)
# Should only execute one query to count the length of the result set.
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.sqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 3)
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.sqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.sqs._cache_is_full(), False)
results = self.sqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test___and__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((foo) AND (bar))')
# Now for something more complex...
sqs3 = self.sqs.exclude(
title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar')
sqs = sqs3 & sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 3)
self.assertEqual(
sqs.query.build_query(),
u'(NOT (title:(moof)) AND ((foo) OR (baz)) AND (bar))')
def test___or__(self):
sqs1 = self.sqs.filter(content='foo')
sqs2 = self.sqs.filter(content='bar')
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(sqs.query.build_query(), u'((foo) OR (bar))')
# Now for something more complex...
sqs3 = self.sqs.exclude(
title='moof').filter(SQ(content='foo') | SQ(content='baz'))
sqs4 = self.sqs.filter(content='bar').models(MockModel)
sqs = sqs3 | sqs4
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
self.assertEqual(
sqs.query.build_query(),
u'((NOT (title:(moof)) AND ((foo) OR (baz))) OR (bar))')
def test_auto_query(self):
# Ensure bits in exact matches get escaped properly as well.
# This will break horrifically if escaping isn't working.
sqs = self.sqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__contains="pants:rule">')
self.assertEqual(sqs.query.build_query(), u'("pants\\:rule")')
self.assertEqual(len(sqs), 0)
# Regressions
def test_regression_proper_start_offsets(self):
sqs = self.sqs.filter(text='index')
self.assertNotEqual(sqs.count(), 0)
id_counts = {}
for item in sqs:
if item.id in id_counts:
id_counts[item.id] += 1
else:
id_counts[item.id] = 1
for key, value in id_counts.items():
if value > 1:
self.fail(
"Result with id '%s' seen more than once in the results." %
key)
def test_regression_raw_search_breaks_slicing(self):
sqs = self.sqs.raw_search('text: index')
page_1 = [result.pk for result in sqs[0:10]]
page_2 = [result.pk for result in sqs[10:20]]
for pk in page_2:
if pk in page_1:
self.fail(
"Result with id '%s' seen more than once in the results." %
pk)
# RelatedSearchQuerySet Tests
def test_related_load_all(self):
sqs = self.rsqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue(len(sqs) > 0)
self.assertEqual(
sqs[0].object.foo,
u"Registering indexes in Haystack is very similar to registering models and ``ModelAdmin`` classes in the `Django admin site`_. If you want to override the default indexing behavior for your model you can specify your own ``SearchIndex`` class. This is useful for ensuring that future-dated or non-live content is not indexed and searchable. Our ``Note`` model has a ``pub_date`` field, so let's update our code to include our own ``SearchIndex`` to exclude indexing future-dated notes:"
)
def test_related_load_all_queryset(self):
sqs = self.rsqs.load_all()
self.assertEqual(len(sqs._load_all_querysets), 0)
sqs = sqs.load_all_queryset(MockModel,
MockModel.objects.filter(id__gt=1))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], range(2, 24))
sqs = sqs.load_all_queryset(MockModel,
MockModel.objects.filter(id__gt=10))
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs._load_all_querysets), 1)
self.assertEqual([obj.object.id for obj in sqs], range(11, 24))
self.assertEqual([obj.object.id for obj in sqs[10:20]], [21, 22, 23])
def test_related_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
sqs = self.rsqs.all()
results = [int(result.pk) for result in sqs]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[1:11]],
[2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(int(results[21].pk), 22)
self.assertEqual(len(connections['default'].queries), 4)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual([int(result.pk) for result in results[20:30]],
[21, 22, 23])
self.assertEqual(len(connections['default'].queries), 4)
def test_related_manual_iter(self):
results = self.rsqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = [int(result.pk) for result in results._manual_iter()]
self.assertEqual(results, range(1, 24))
self.assertEqual(len(connections['default'].queries), 4)
def test_related_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.rsqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
def test_related_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.rsqs._cache_is_full(), False)
results = self.rsqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 5)
def test_quotes_regression(self):
sqs = self.sqs.auto_query(u"44°48'40''N 20°28'32''E")
# Should not have empty terms.
self.assertEqual(sqs.query.build_query(),
u"(44\xb048'40''N 20\xb028'32''E)")
# Should not cause Solr to 500.
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing')
self.assertEqual(sqs.query.build_query(), u'(blazing)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('blazing saddles')
self.assertEqual(sqs.query.build_query(), u'(blazing saddles)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles')
self.assertEqual(sqs.query.build_query(), u'(\\"blazing saddles)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles"')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'saddles"')
self.assertEqual(sqs.query.build_query(), u'(mel "blazing \'saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"')
self.assertEqual(sqs.query.build_query(),
u'(mel "blazing \'\'saddles")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'')
self.assertEqual(sqs.query.build_query(),
u'(mel "blazing \'\'saddles" \')')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing \'\'saddles"\'"')
self.assertEqual(sqs.query.build_query(),
u'(mel "blazing \'\'saddles" \'\\")')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel')
self.assertEqual(sqs.query.build_query(), u'("blazing saddles" mel)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('"blazing saddles" mel brooks')
self.assertEqual(sqs.query.build_query(),
u'("blazing saddles" mel brooks)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" brooks')
self.assertEqual(sqs.query.build_query(),
u'(mel "blazing saddles" brooks)')
self.assertEqual(sqs.count(), 0)
sqs = self.sqs.auto_query('mel "blazing saddles" "brooks')
self.assertEqual(sqs.query.build_query(),
u'(mel "blazing saddles" \\"brooks)')
self.assertEqual(sqs.count(), 0)
def test_query_generation(self):
sqs = self.sqs.filter(
SQ(content=AutoQuery("hello world"))
| SQ(title=AutoQuery("hello world")))
self.assertEqual(sqs.query.build_query(),
u"((hello world) OR title:(hello world))")
def test_result_class(self):
# Assert that we're defaulting to ``SearchResult``.
sqs = self.sqs.all()
self.assertTrue(isinstance(sqs[0], SearchResult))
# Custom class.
sqs = self.sqs.result_class(MockSearchResult).all()
self.assertTrue(isinstance(sqs[0], MockSearchResult))
# Reset to default.
sqs = self.sqs.result_class(None).all()
self.assertTrue(isinstance(sqs[0], SearchResult))
def haystack_search_api(request, format="json", **kwargs):
"""
View that drives the search api
"""
from haystack.inputs import Raw
from haystack.query import SearchQuerySet, SQ
# Retrieve Query Params
id = request.REQUEST.get("id", None)
query = request.REQUEST.get('q', None)
category = request.REQUEST.get("category", None)
limit = int(
request.REQUEST.get(
"limit", getattr(settings, "HAYSTACK_SEARCH_RESULTS_PER_PAGE",
20)))
startIndex = int(request.REQUEST.get("startIndex", 0))
sort = request.REQUEST.get("sort", "relevance")
type_facets = request.REQUEST.get("type", None)
format = request.REQUEST.get("format", format)
date_start = request.REQUEST.get("start_date", None)
date_end = request.REQUEST.get("end_date", None)
keyword = request.REQUEST.get("kw", None)
service = request.REQUEST.get("service", None)
local = request.REQUEST.get("local", None)
# Geospatial Elements
bbox = request.REQUEST.get("extent", None)
ts = time()
sqs = SearchQuerySet()
limit = min(limit, 500)
# Filter by ID
if id:
sqs = sqs.narrow("django_id:%s" % id)
# Filter by Type and subtype
if type_facets is not None:
type_facets = type_facets.replace("owner", "user").split(",")
subtype_facets = ["vector", "raster"]
types = []
subtypes = []
for type in type_facets:
if type in ["map", "layer", "user", "document", "group"]:
# Type is one of our Major Types (not a sub type)
types.append(type)
elif type in subtype_facets:
subtypes.append(type)
if len(subtypes) > 0:
for sub_type in subtype_facets:
if sub_type not in subtypes:
sqs = sqs.exclude(subtype='%s' % sub_type)
if len(types) > 0:
sqs = sqs.narrow("type:%s" % ','.join(map(str, types)))
# Filter by Query Params
# haystack bug? if boosted fields aren't included in the
# query, then the score won't be affected by the boost
if query:
if query.startswith('"') or query.startswith('\''):
#Match exact phrase
phrase = query.replace('"', '')
sqs = sqs.filter(
SQ(title__exact=phrase) | SQ(abstract__exact=phrase)
| SQ(content__exact=phrase))
else:
words = query.split()
for word in range(0, len(words)):
if word == 0:
sqs = sqs.filter(
SQ(title=Raw(words[word]))
| SQ(abstract=Raw(words[word]))
| SQ(content=Raw(words[word])))
elif words[word] in ["AND", "OR"]:
pass
elif words[word - 1] == "OR": #previous word OR this word
sqs = sqs.filter_or(
SQ(title=Raw(words[word]))
| SQ(abstract=Raw(words[word]))
| SQ(content=Raw(words[word])))
else: #previous word AND this word
sqs = sqs.filter(
SQ(title=Raw(words[word]))
| SQ(abstract=Raw(words[word]))
| SQ(content=Raw(words[word])))
# filter by cateory
if category:
sqs = sqs.narrow('category:%s' % category)
#filter by keyword
if keyword:
sqs = sqs.narrow('keywords:%s' % keyword)
if date_start:
sqs = sqs.filter(SQ(date__gte=date_start))
if date_end:
sqs = sqs.filter(SQ(date__lte=date_end))
"""
### Code to filter on temporal extent start/end dates instead
if date_start or date_end:
#Exclude results with no dates at all
sqs = sqs.filter(
SQ(temporal_extent_start=Raw("[* TO *]")) | SQ(temporal_extent_end=Raw("[* TO *]"))
)
if temporal_start and temporal_end:
#Return anything with a start date < date_end or an end date > date_start
sqs = sqs.filter(
SQ(temporal_extent_end__gte=date_start) | SQ(temporal_extent_start__lte=date_end)
)
elif temporal_start:
#Exclude anything with an end date <date_start
sqs = sqs.exclude(
SQ(temporal_extent_end__lte=date_start)
)
elif temporal_end:
#Exclude anything with a start date > date_end
sqs = sqs.exclude(
SQ(temporal_extent_start__gte=date_end)
)
"""
if bbox:
left, right, bottom, top = bbox.split(',')
sqs = sqs.filter(
# first check if the bbox has at least one point inside the window
SQ(bbox_left__gte=left) & SQ(bbox_left__lte=right)
& SQ(bbox_top__gte=bottom) & SQ(bbox_top__lte=top)
| #check top_left is inside the window
SQ(bbox_right__lte=right) & SQ(bbox_right__gte=left)
& SQ(bbox_top__lte=top) & SQ(bbox_top__gte=bottom)
| #check top_right is inside the window
SQ(bbox_bottom__gte=bottom) & SQ(bbox_bottom__lte=top)
& SQ(bbox_right__lte=right) & SQ(bbox_right__gte=left)
| #check bottom_right is inside the window
SQ(bbox_top__lte=top) & SQ(bbox_top__gte=bottom)
& SQ(bbox_left__gte=left) & SQ(bbox_left__lte=right)
| #check bottom_left is inside the window
# then check if the bbox is including the window
SQ(bbox_left__lte=left) & SQ(bbox_right__gte=right)
& SQ(bbox_bottom__lte=bottom) & SQ(bbox_top__gte=top))
# Filter by permissions
'''
### Takes too long with many results.
### Instead, show all results but disable links on restricted ones.
for i, result in enumerate(sqs):
if result.type == 'layer':
if not request.user.has_perm('layers.view_layer',obj = result.object):
sqs = sqs.exclude(id = result.id)
if result.type == 'map':
if not request.user.has_perm('maps.view_map',obj = result.object):
sqs = sqs.exclude(id = result.id)
'''
#filter by service
'''
if service:
sqs = sqs.narrow('service:%s' % service)
if local:
sqs = sqs.narrow('local:%s' % local)
'''
# Apply Sort
# TODO: Handle for Revised sort types
# [relevance, alphabetically, rating, created, updated, popularity]
if sort.lower() == "newest":
sqs = sqs.order_by("-modified")
elif sort.lower() == "oldest":
sqs = sqs.order_by("modified")
elif sort.lower() == "alphaaz":
sqs = sqs.order_by("title_sortable")
elif sort.lower() == "alphaza":
sqs = sqs.order_by("-title_sortable")
elif sort.lower() == "popularity":
sqs = sqs.order_by("-popular_count")
else:
sqs = sqs.order_by("-_score")
# Setup Search Results
results = []
items = []
# Build the result based on the limit
for i, result in enumerate(sqs[startIndex:startIndex + limit]):
logger.info(result)
data = result.get_stored_fields()
resource = None
if "modified" in data:
data["modified"] = data["modified"].strftime(
"%Y-%m-%dT%H:%M:%S.%f")
if "temporal_extent_start" in data and data[
"temporal_extent_start"] is not None:
data["temporal_extent_start"] = data[
"temporal_extent_start"].strftime("%Y-%m-%dT%H:%M:%S.%f")
if "temporal_extent_end" in data and data[
"temporal_extent_end"] is not None:
data["temporal_extent_end"] = data["temporal_extent_end"].strftime(
"%Y-%m-%dT%H:%M:%S.%f")
if data['type'] == "map":
resource = MapNormalizer(Map.objects.get(pk=data['oid']))
elif data['type'] == "layer":
resource = LayerNormalizer(Layer.objects.get(pk=data['oid']))
elif data['type'] == "user":
resource = OwnerNormalizer(Profile.objects.get(pk=data['oid']))
elif data['type'] == "document":
resource = DocumentNormalizer(Document.objects.get(pk=data['oid']))
elif data[
'type'] == "group" and "geonode.contrib.groups" in settings.INSTALLED_APPS:
resource = GroupNormalizer(Group.objects.get(pk=data['oid']))
if resource:
resource.rating = data["rating"] if "rating" in data else 0
results.append(data)
items.append(resource)
# Setup Facet Counts
sqs = sqs.facet("type").facet("subtype")
sqs = sqs.facet('category')
sqs = sqs.facet('keywords')
sqs = sqs.facet('service')
sqs = sqs.facet('local')
facet_counts = sqs.facet_counts()
# Prepare Search Results
data = {
"success":
True,
"total":
sqs.count(),
"query_info": {
"q": query,
"startIndex": startIndex,
"limit": limit,
"sort": sort,
"type": type_facets,
},
"results":
results,
"facets":
dict(
facet_counts.get("fields")['type'] +
facet_counts.get('fields')['subtype']) if sqs.count() > 0 else [],
"categories": {
facet[0]: facet[1]
for facet in facet_counts.get('fields')['category']
} if sqs.count() > 0 else {},
"keywords": {
facet[0]: facet[1]
for facet in facet_counts.get('fields')['keywords']
} if sqs.count() > 0 else {},
}
# Return Results
ts1 = time() - ts
if format == "html": #Send to search/explore page
return data, items
elif format == "raw":
return HttpResponse(json.dumps(data), mimetype="application/json")
else:
query = query_from_request(request, kwargs)
return _search_json(query, items, data["facets"], ts1)
class SearchQuerySetTestCase(TestCase):
fixtures = ['bulk_data.json']
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections['default']._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections['default']._index = self.ui
# Update the "index".
backend = connections['default'].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
self.old_debug = settings.DEBUG
settings.DEBUG = True
reset_search_queries()
def tearDown(self):
# Restore.
connections['default']._index = self.old_unified_index
settings.DEBUG = self.old_debug
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(
repr(self.msqs),
"[<SearchResult: core.mockmodel (pk=u'1')>, <SearchResult: core.mockmodel (pk=u'2')>, <SearchResult: core.mockmodel (pk=u'3')>, <SearchResult: core.mockmodel (pk=u'4')>, <SearchResult: core.mockmodel (pk=u'5')>, <SearchResult: core.mockmodel (pk=u'6')>, <SearchResult: core.mockmodel (pk=u'7')>, <SearchResult: core.mockmodel (pk=u'8')>, <SearchResult: core.mockmodel (pk=u'9')>, <SearchResult: core.mockmodel (pk=u'10')>, <SearchResult: core.mockmodel (pk=u'11')>, <SearchResult: core.mockmodel (pk=u'12')>, <SearchResult: core.mockmodel (pk=u'13')>, <SearchResult: core.mockmodel (pk=u'14')>, <SearchResult: core.mockmodel (pk=u'15')>, <SearchResult: core.mockmodel (pk=u'16')>, <SearchResult: core.mockmodel (pk=u'17')>, <SearchResult: core.mockmodel (pk=u'18')>, <SearchResult: core.mockmodel (pk=u'19')>, '...(remaining elements truncated)...']"
)
self.assertEqual(len(connections['default'].queries), 1)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in msqs]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections['default'].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]],
[res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections['default'].queries), 1)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections['default'].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(check, [
u'1', u'2', u'3', u'4', u'5', u'6', u'7', u'8', u'9', u'10', u'11',
u'12', u'13', u'14', u'15', u'16', u'17', u'18', u'19', u'20',
u'21', u'22', u'23'
])
self.assertEqual(len(connections['default'].queries), 3)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections['default']._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, [u'sometext', u'1234'])
self.assertEqual(len(connections['default'].queries), 1)
connections['default']._index = old_ui
def test_fill_cache(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
results = self.msqs.all()
self.assertEqual(len(results._result_cache), 0)
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 10)
self.assertEqual(len(connections['default'].queries), 1)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual(len(connections['default'].queries), 2)
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
sqs = SearchQuerySet().all()
sqs.query.backend = MixedMockSearchBackend('default')
results = sqs
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 0)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [])
self.assertEqual(len(connections['default'].queries), 0)
results._fill_cache(0, 10)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 9)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10])
self.assertEqual(len(connections['default'].queries), 2)
results._fill_cache(10, 20)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 17)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20])
self.assertEqual(len(connections['default'].queries), 4)
results._fill_cache(20, 30)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 20)
self.assertEqual([
int(result.pk)
for result in results._result_cache if result is not None
], [
1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 15, 16, 17, 18, 19, 20, 21, 22,
23
])
self.assertEqual(len(connections['default'].queries), 6)
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections['default'].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = [result for result in results]
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections['default'].queries), 3)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content='foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue('foo' in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections['default']._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections['default']._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost('foo', 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content='Indx')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion('indexy'), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search('foo')), 23)
self.assertEqual(
len(
self.msqs.raw_search(
'(content__exact:hello AND content__exact:world)')), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend('charpk')
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# If nothing is handled, you get nothing.
old_ui = connections['default']._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections['default']._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections['default']._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections['default']._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections['default']._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend('default')
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(
len([
result for result in results._result_cache
if result is not None
]), 2)
# Restore.
connections['default']._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query('test search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND (content__contains=test AND content__contains=search AND NOT (content__contains=stuff))>'
)
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND (content__exact=my thing AND content__contains=test AND content__contains=search AND NOT (content__contains=stuff))>'
)
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
"<SQ: AND (content__exact=my thing AND content__contains=test AND content__contains=search AND content__contains='moar AND content__contains=quotes' AND NOT (content__contains=stuff))>"
)
sqs = self.msqs.auto_query(
'test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND (content__exact=my thing AND content__contains=test AND content__contains=search AND content__contains=\'moar AND content__contains=quotes\' AND content__contains="foo AND NOT (content__contains=stuff))>'
)
sqs = self.msqs.auto_query('test - stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND (content__contains=test AND content__contains=- AND content__contains=stuff)>'
)
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter),
'<SQ: AND content__exact=pants:rule>')
# Now with a different fieldname
sqs = self.msqs.auto_query('test search -stuff', fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND (title__contains=test AND title__contains=search AND NOT (title__contains=stuff))>'
)
sqs = self.msqs.auto_query('test "my thing" search -stuff',
fieldname='title')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND (title__exact=my thing AND title__contains=test AND title__contains=search AND NOT (title__contains=stuff))>'
)
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest('pub_date'), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet('foo')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet('foo').facet('bar')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='smarblaph')
self.fail()
except FacetingError, e:
self.assertEqual(
str(e),
"The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second."
)
sqs = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet('foo',
start_date=datetime.date(2008, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='month').date_facet(
'bar',
start_date=datetime.date(2007, 2, 25),
end_date=datetime.date(2009, 2, 25),
gap_by='year')
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
class SearchQuerySetTestCase(TestCase):
fixtures = ["base_data.json", "bulk_data.json"]
def setUp(self):
super(SearchQuerySetTestCase, self).setUp()
# Stow.
self.old_unified_index = connections["default"]._index
self.ui = UnifiedIndex()
self.bmmsi = BasicMockModelSearchIndex()
self.cpkmmsi = CharPKMockModelSearchIndex()
self.ui.build(indexes=[self.bmmsi, self.cpkmmsi])
connections["default"]._index = self.ui
# Update the "index".
backend = connections["default"].get_backend()
backend.clear()
backend.update(self.bmmsi, MockModel.objects.all())
self.msqs = SearchQuerySet()
# Stow.
reset_search_queries()
def tearDown(self):
# Restore.
connections["default"]._index = self.old_unified_index
super(SearchQuerySetTestCase, self).tearDown()
def test_len(self):
self.assertEqual(len(self.msqs), 23)
def test_repr(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertRegexpMatches(
repr(self.msqs),
r"^<SearchQuerySet: query=<test_haystack.mocks.MockSearchQuery object" r" at 0x[0-9A-Fa-f]+>, using=None>$",
)
def test_iter(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
msqs = self.msqs.all()
results = [int(res.pk) for res in iter(msqs)]
self.assertEqual(results, [res.pk for res in MOCK_SEARCH_RESULTS[:23]])
self.assertEqual(len(connections["default"].queries), 3)
def test_slice(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual([int(res.pk) for res in results[1:11]], [res.pk for res in MOCK_SEARCH_RESULTS[1:11]])
self.assertEqual(len(connections["default"].queries), 1)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
results = self.msqs.all()
self.assertEqual(int(results[22].pk), MOCK_SEARCH_RESULTS[22].pk)
self.assertEqual(len(connections["default"].queries), 1)
def test_manual_iter(self):
results = self.msqs.all()
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
check = [result.pk for result in results._manual_iter()]
self.assertEqual(
check,
[
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12",
"13",
"14",
"15",
"16",
"17",
"18",
"19",
"20",
"21",
"22",
"23",
],
)
self.assertEqual(len(connections["default"].queries), 3)
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
# Test to ensure we properly fill the cache, even if we get fewer
# results back (not a handled model) than the hit count indicates.
# This will hang indefinitely if broken.
old_ui = self.ui
self.ui.build(indexes=[self.cpkmmsi])
connections["default"]._index = self.ui
self.cpkmmsi.update()
results = self.msqs.all()
loaded = [result.pk for result in results._manual_iter()]
self.assertEqual(loaded, ["sometext", "1234"])
self.assertEqual(len(connections["default"].queries), 1)
connections["default"]._index = old_ui
def test_cache_is_full(self):
reset_search_queries()
self.assertEqual(len(connections["default"].queries), 0)
self.assertEqual(self.msqs._cache_is_full(), False)
results = self.msqs.all()
fire_the_iterator_and_fill_cache = list(results)
self.assertEqual(23, len(fire_the_iterator_and_fill_cache))
self.assertEqual(results._cache_is_full(), True)
self.assertEqual(len(connections["default"].queries), 4)
def test_all(self):
sqs = self.msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
def test_filter(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_exclude(self):
sqs = self.msqs.exclude(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_order_by(self):
sqs = self.msqs.order_by("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertTrue("foo" in sqs.query.order_by)
def test_models(self):
# Stow.
old_unified_index = connections["default"]._index
ui = UnifiedIndex()
bmmsi = BasicMockModelSearchIndex()
bammsi = BasicAnotherMockModelSearchIndex()
ui.build(indexes=[bmmsi, bammsi])
connections["default"]._index = ui
msqs = SearchQuerySet()
sqs = msqs.all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 0)
sqs = msqs.models(MockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
sqs = msqs.models(MockModel, AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 2)
# This will produce a warning.
ui.build(indexes=[bmmsi])
sqs = msqs.models(AnotherMockModel)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.models), 1)
def test_result_class(self):
sqs = self.msqs.all()
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
# Custom class.
class IttyBittyResult(object):
pass
sqs = self.msqs.result_class(IttyBittyResult)
self.assertTrue(issubclass(sqs.query.result_class, IttyBittyResult))
# Reset to default.
sqs = self.msqs.result_class(None)
self.assertTrue(issubclass(sqs.query.result_class, SearchResult))
def test_boost(self):
sqs = self.msqs.boost("foo", 10)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.boost.keys()), 1)
def test_highlight(self):
sqs = self.msqs.highlight()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.highlight, True)
def test_spelling_override(self):
sqs = self.msqs.filter(content="not the spellchecking query")
self.assertEqual(sqs.query.spelling_query, None)
sqs = self.msqs.set_spelling_query("override")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.query.spelling_query, "override")
def test_spelling_suggestions(self):
# Test the case where spelling support is disabled.
sqs = self.msqs.filter(content="Indx")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(sqs.spelling_suggestion(), None)
self.assertEqual(sqs.spelling_suggestion("indexy"), None)
def test_raw_search(self):
self.assertEqual(len(self.msqs.raw_search("foo")), 23)
self.assertEqual(len(self.msqs.raw_search("(content__exact:hello AND content__exact:world)")), 23)
def test_load_all(self):
# Models with character primary keys.
sqs = SearchQuerySet()
sqs.query.backend = CharPKMockSearchBackend("charpk")
results = sqs.load_all().all()
self.assertEqual(len(results._result_cache), 0)
results._fill_cache(0, 2)
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# If nothing is handled, you get nothing.
old_ui = connections["default"]._index
ui = UnifiedIndex()
ui.build(indexes=[])
connections["default"]._index = ui
sqs = self.msqs.load_all()
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs), 0)
connections["default"]._index = old_ui
# For full tests, see the solr_backend.
def test_load_all_read_queryset(self):
# Stow.
old_ui = connections["default"]._index
ui = UnifiedIndex()
gafmmsi = GhettoAFifthMockModelSearchIndex()
ui.build(indexes=[gafmmsi])
connections["default"]._index = ui
gafmmsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# The deleted result isn't returned
self.assertEqual(len([result for result in results._result_cache if result is not None]), 1)
# Register a SearchIndex with a read_queryset that returns deleted items
rqstsi = TextReadQuerySetTestSearchIndex()
ui.build(indexes=[rqstsi])
rqstsi.update()
sqs = SearchQuerySet()
results = sqs.load_all().all()
results.query.backend = ReadQuerySetMockSearchBackend("default")
results._fill_cache(0, 2)
# Both the deleted and not deleted items are returned
self.assertEqual(len([result for result in results._result_cache if result is not None]), 2)
# Restore.
connections["default"]._index = old_ui
def test_auto_query(self):
sqs = self.msqs.auto_query("test search -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__content=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__content=test "my thing" search -stuff>')
sqs = self.msqs.auto_query("test \"my thing\" search 'moar quotes' -stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter), "<SQ: AND content__content=test \"my thing\" search 'moar quotes' -stuff>"
)
sqs = self.msqs.auto_query('test "my thing" search \'moar quotes\' "foo -stuff')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(
repr(sqs.query.query_filter),
'<SQ: AND content__content=test "my thing" search \'moar quotes\' "foo -stuff>',
)
sqs = self.msqs.auto_query("test - stuff")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND content__content=test - stuff>")
# Ensure bits in exact matches get escaped properly as well.
sqs = self.msqs.auto_query('"pants:rule"')
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND content__content="pants:rule">')
# Now with a different fieldname
sqs = self.msqs.auto_query("test search -stuff", fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), "<SQ: AND title__content=test search -stuff>")
sqs = self.msqs.auto_query('test "my thing" search -stuff', fieldname="title")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(repr(sqs.query.query_filter), '<SQ: AND title__content=test "my thing" search -stuff>')
def test_count(self):
self.assertEqual(self.msqs.count(), 23)
def test_facet_counts(self):
self.assertEqual(self.msqs.facet_counts(), {})
def test_best_match(self):
self.assertTrue(isinstance(self.msqs.best_match(), SearchResult))
def test_latest(self):
self.assertTrue(isinstance(self.msqs.latest("pub_date"), SearchResult))
def test_more_like_this(self):
mock = MockModel()
mock.id = 1
self.assertEqual(len(self.msqs.more_like_this(mock)), 23)
def test_facets(self):
sqs = self.msqs.facet("foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.facets), 1)
sqs2 = self.msqs.facet("foo").facet("bar")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.facets), 2)
def test_date_facets(self):
try:
sqs = self.msqs.date_facet(
"foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="smarblaph"
)
self.fail()
except FacetingError as e:
self.assertEqual(
str(e), "The gap_by ('smarblaph') must be one of the following: year, month, day, hour, minute, second."
)
sqs = self.msqs.date_facet(
"foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month"
)
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.date_facets), 1)
sqs2 = self.msqs.date_facet(
"foo", start_date=datetime.date(2008, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="month"
).date_facet("bar", start_date=datetime.date(2007, 2, 25), end_date=datetime.date(2009, 2, 25), gap_by="year")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.date_facets), 2)
def test_query_facets(self):
sqs = self.msqs.query_facet("foo", "[bar TO *]")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_facets), 1)
sqs2 = self.msqs.query_facet("foo", "[bar TO *]").query_facet("bar", "[100 TO 499]")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.query_facets), 2)
# Test multiple query facets on a single field
sqs3 = (
self.msqs.query_facet("foo", "[bar TO *]")
.query_facet("bar", "[100 TO 499]")
.query_facet("foo", "[1000 TO 1499]")
)
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.query_facets), 3)
def test_stats(self):
sqs = self.msqs.stats_facet("foo", "bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.stats), 1)
sqs2 = self.msqs.stats_facet("foo", "bar").stats_facet("foo", "baz")
self.assertTrue(isinstance(sqs2, SearchQuerySet))
self.assertEqual(len(sqs2.query.stats), 1)
sqs3 = self.msqs.stats_facet("foo", "bar").stats_facet("moof", "baz")
self.assertTrue(isinstance(sqs3, SearchQuerySet))
self.assertEqual(len(sqs3.query.stats), 2)
def test_narrow(self):
sqs = self.msqs.narrow("foo:moof")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.narrow_queries), 1)
def test_clone(self):
results = self.msqs.filter(foo="bar", foo__lt="10")
clone = results._clone()
self.assertTrue(isinstance(clone, SearchQuerySet))
self.assertEqual(str(clone.query), str(results.query))
self.assertEqual(clone._result_cache, [])
self.assertEqual(clone._result_count, None)
self.assertEqual(clone._cache_full, False)
self.assertEqual(clone._using, results._using)
def test_using(self):
sqs = SearchQuerySet(using="default")
self.assertNotEqual(sqs.query, None)
self.assertEqual(sqs.query._using, "default")
def test_chaining(self):
sqs = self.msqs.filter(content="foo")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
# A second instance should inherit none of the changes from above.
sqs = self.msqs.filter(content="bar")
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 1)
def test_none(self):
sqs = self.msqs.none()
self.assertTrue(isinstance(sqs, EmptySearchQuerySet))
self.assertEqual(len(sqs), 0)
def test___and__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 & sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test___or__(self):
sqs1 = self.msqs.filter(content="foo")
sqs2 = self.msqs.filter(content="bar")
sqs = sqs1 | sqs2
self.assertTrue(isinstance(sqs, SearchQuerySet))
self.assertEqual(len(sqs.query.query_filter), 2)
def test_and_or(self):
"""
Combining AND queries with OR should give
AND(OR(a, b), OR(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter(content="oof")
sqs2 = self.msqs.filter(content="bar").filter(content="rab")
sqs = sqs1 | sqs2
self.assertEqual(sqs.query.query_filter.connector, "OR")
self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter))
def test_or_and(self):
"""
Combining OR queries with AND should give
OR(AND(a, b), AND(c, d))
"""
sqs1 = self.msqs.filter(content="foo").filter_or(content="oof")
sqs2 = self.msqs.filter(content="bar").filter_or(content="rab")
sqs = sqs1 & sqs2
self.assertEqual(sqs.query.query_filter.connector, "AND")
self.assertEqual(repr(sqs.query.query_filter.children[0]), repr(sqs1.query.query_filter))
self.assertEqual(repr(sqs.query.query_filter.children[1]), repr(sqs2.query.query_filter))
def map_search_endpoint(request, filter_group_id=None):
""" Maps API search endpoint using haystack search results. For parameters see ``MAP_SEARCH_PARAMETERS``
returns JSON with the contents of type ``HaystackMapResult``
@param filter_group_id: Will filter all items by group relation, where applicable
(i.e. users are filtered by group memberships for that group, events as events in that group)
"""
implicit_ignore_location = not any([
loc_param in request.GET
for loc_param in ['sw_lon', 'sw_lat', 'ne_lon', 'ne_lat']
])
params = _collect_parameters(request.GET, MAP_SEARCH_PARAMETERS)
query = force_text(params['q'])
limit = params['limit']
page = params['page']
item_id = params['item']
if params.get('cloudfiles', False):
return map_cloudfiles_endpoint(request, query, limit, page)
# TODO: set to params['external'] after the external switch button is in frontend!
external = settings.COSINNUS_EXTERNAL_CONTENT_ENABLED
prefer_own_portal = getattr(settings, 'MAP_API_HACKS_PREFER_OWN_PORTAL',
False)
if not is_number(limit) or limit < 0:
return HttpResponseBadRequest(
'``limit`` param must be a positive number or 0!')
limit = min(limit, SERVER_SIDE_SEARCH_LIMIT)
if not is_number(page) or page < 0:
return HttpResponseBadRequest(
'``page`` param must be a positive number or 0!')
# filter for requested model types
model_list = [
klass for klass, param_name in list(SEARCH_MODEL_NAMES.items())
if params.get(param_name, False)
]
sqs = SearchQuerySet().models(*model_list)
# filter for map bounds (Points are constructed ith (lon, lat)!!!)
if not params['ignore_location'] and not implicit_ignore_location:
sqs = sqs.within('location', Point(params['sw_lon'], params['sw_lat']),
Point(params['ne_lon'], params['ne_lat']))
# filter for user's own content
if params['mine'] and request.user.is_authenticated:
user_id = request.user.id
sqs = sqs.filter_and(
Q(creator=user_id) | Q(user_id=user_id) | Q(group_members=user_id))
# filter for search terms
if query:
sqs = sqs.auto_query(query)
# group-filtered-map view for on-group pages
if filter_group_id:
group = get_object_or_None(get_cosinnus_group_model(),
id=filter_group_id)
if group:
filtered_groups = [filter_group_id]
# get child projects of this group
filtered_groups += [
subproject.id for subproject in group.get_children()
if subproject.is_active
]
sqs = sqs.filter_and(
Q(membership_groups__in=filtered_groups)
| Q(group__in=filtered_groups))
# filter topics
topics = ensure_list_of_ints(params.get('topics', ''))
if topics:
sqs = sqs.filter_and(mt_topics__in=topics)
if settings.COSINNUS_ENABLE_SDGS:
sdgs = ensure_list_of_ints(params.get('sdgs', ''))
if sdgs:
sqs = sqs.filter_and(sdgs__in=sdgs)
if settings.COSINNUS_MANAGED_TAGS_ENABLED:
managed_tags = ensure_list_of_ints(params.get('managed_tags', ''))
if managed_tags:
sqs = sqs.filter_and(managed_tags__in=managed_tags)
# filter for portal visibility
sqs = filter_searchqueryset_for_portal(
sqs,
restrict_multiportals_to_current=prefer_own_portal,
external=external)
# filter for read access by this user
sqs = filter_searchqueryset_for_read_access(sqs, request.user)
# filter events by upcoming status and exclude hidden proxies
if params['events'] and Event is not None:
sqs = filter_event_searchqueryset_by_upcoming(sqs).exclude(
is_hidden_group_proxy=True)
# filter all default user groups if the new dashboard is being used (they count as "on plattform" and aren't shown)
if getattr(settings, 'COSINNUS_USE_V2_DASHBOARD', False):
sqs = sqs.exclude(is_group_model=True,
slug__in=get_default_user_group_slugs())
# kip score sorting and only rely on natural ordering?
skip_score_sorting = False
# if we hae no query-boosted results, use *only* our custom sorting (haystack's is very random)
if not query:
sort_args = ['-local_boost']
# if we only look at conferences, order them by their from_date, future first!
if prefer_own_portal:
sort_args = ['-portal'] + sort_args
"""
# this would be the way to force-sort a content type by a natural ordering instead of score if its the only type being shown
if params.get('conferences', False) and sum([1 if params.get(content_key, False) else 0 for content_key in MAP_CONTENT_TYPE_SEARCH_PARAMETERS.keys()]) == 1:
sort_args = ['-from_date'] + sort_args
skip_score_sorting = True
sqs = sqs.order_by(*sort_args)
"""
# sort results into one list per model
total_count = sqs.count()
sqs = sqs[limit * page:limit * (page + 1)]
results = []
for i, result in enumerate(sqs):
if skip_score_sorting:
# if we skip score sorting and only rely on the natural ordering, we make up fake high scores
result.score = 100000 - (limit * page) - i
elif not query:
# if we hae no query-boosted results, use *only* our custom sorting (haystack's is very random)
result.score = result.local_boost
if prefer_own_portal and is_number(result.portal) and int(
result.portal) == CosinnusPortal.get_current().id:
result.score += 100.0
results.append(HaystackMapResult(result, user=request.user))
# if the requested item (direct select) is not in the queryset snippet
# (might happen because of an old URL), then mix it in as first item and drop the last
if item_id:
item_id = str(item_id)
if not any([res['id'] == item_id for res in results]):
item_result = get_searchresult_by_itemid(item_id, request.user)
if item_result:
results = [HaystackMapResult(item_result, user=request.user)
] + results[:-1]
page_obj = None
if results:
page_obj = {
'index': page,
'count': len(results),
'total_count': total_count,
'start': (limit * page) + 1,
'end': (limit * page) + len(results),
'has_next': total_count > (limit * (page + 1)),
'has_previous': page > 0,
}
data = {
'results': results,
'page': page_obj,
}
return JsonResponse(data)
