def test_get_indexes(self):
"""Test get_indexes always returns a list of strings."""
# Pulls it from ES_INDEXES (list of strings).
s = S(FakeDjangoMappingType)
eq_(s.get_indexes(), ['elasticutilstest'])
# Pulls it from ES_INDEXES (string).
old_indexes = settings.ES_INDEXES
try:
settings.ES_INDEXES = {'default': 'elasticutilstest'}
s = S(FakeDjangoMappingType)
eq_(s.get_indexes(), ['elasticutilstest'])
finally:
settings.ES_INDEXES = old_indexes
# Pulls from indexes.
s = S(FakeDjangoMappingType).indexes('footest')
eq_(s.get_indexes(), ['footest'])
s = S(FakeDjangoMappingType).indexes('footest', 'footest2')
eq_(s.get_indexes(), ['footest', 'footest2'])
s = S(FakeDjangoMappingType).indexes('footest').indexes('footest2')
eq_(s.get_indexes(), ['footest2'])
def search(cls, query, include_non_vouched=False):
"""Sensible default search for UserProfiles."""
query = query.lower().strip()
fields = ('username', 'bio__text', 'email', 'ircname', 'country__text',
'country__text_phrase', 'region__text',
'region__text_phrase', 'city__text', 'city__text_phrase',
'fullname__text', 'fullname__text_phrase',
'fullname__prefix', 'fullname__fuzzy'
'groups__text')
if query:
q = dict((field, query) for field in fields)
s = (S(cls).boost(fullname__text_phrase=5,
username=5,
email=5,
ircname=5,
fullname__text=4,
country__text_phrase=4,
region__text_phrase=4,
city__text_phrase=4,
fullname__prefix=3,
fullname__fuzzy=2,
bio__text=2).query(or_=q))
else:
s = S(cls)
s = s.order_by('_score', 'name')
if not include_non_vouched:
s = s.filter(is_vouched=True)
return s
def test_filter_or_3(self):
eq_(len(S(FakeDjangoMappingType).filter(F(tag='awesome') | F(tag='boat') |
F(tag='boring'))), 5)
eq_(len(S(FakeDjangoMappingType).filter(or_={'foo': 'bar',
'or_': {'tag': 'boat',
'width': '5'}
})), 3)
def test_filter_empty_f(self):
eq_(len(S(FakeDjangoMappingType).filter(F() | F(tag='awesome'))), 3)
eq_(len(S(FakeDjangoMappingType).filter(F() & F(tag='awesome'))), 3)
eq_(len(S(FakeDjangoMappingType).filter(F() | F() | F(tag='awesome'))),
3)
eq_(len(S(FakeDjangoMappingType).filter(F() & F() & F(tag='awesome'))),
3)
eq_(len(S(FakeDjangoMappingType).filter(F())), 5)
def test_filter_and(self):
eq_(len(S(FakeDjangoMappingType).filter(tag='awesome', foo='bar')), 1)
eq_(
len(
S(FakeDjangoMappingType).filter(tag='awesome').filter(
foo='bar')), 1)
eq_(
len(
S(FakeDjangoMappingType).filter(
F(tag='awesome') & F(foo='bar'))), 1)
def test_facet_raw(self):
qs = S(FakeDjangoMappingType).facet_raw(tags={'terms': {'field': 'tag'}})
eq_(facet_counts_dict(qs, 'tags'),
dict(awesome=3, boring=1, boat=1))
qs = (S(FakeDjangoMappingType)
.query(foo='car')
.facet_raw(tags={'terms': {'field': 'tag'}}))
eq_(facet_counts_dict(qs, 'tags'),
{'awesome': 2})
def test_filter_not(self):
eq_(len(S(FakeDjangoMappingType).filter(~F(tag='awesome'))), 2)
eq_(
len(
S(FakeDjangoMappingType).filter(~(F(tag='boring')
| F(tag='boat')))), 3)
eq_(
len(
S(FakeDjangoMappingType).filter(~F(tag='boat')).filter(~F(
foo='bar'))), 3)
eq_(len(S(FakeDjangoMappingType).filter(~F(tag='boat', foo='barf'))),
5)
def test_get_doctypes(self):
"""Test get_doctypes always returns a list of strings."""
# Pulls from ._meta.db_table.
s = S(FakeDjangoMappingType)
eq_(s.get_doctypes(), ['fake'])
# Pulls from doctypes.
s = S(FakeDjangoMappingType).doctypes('footype')
eq_(s.get_doctypes(), ['footype'])
s = S(FakeDjangoMappingType).doctypes('footype', 'footype2')
eq_(s.get_doctypes(), ['footype', 'footype2'])
s = S(FakeDjangoMappingType).doctypes('footype').doctypes('footype2')
eq_(s.get_doctypes(), ['footype2'])
def app_search(request):
results = []
q = request.GET.get('q', u'').lower().strip()
fields = ('name', 'app_slug')
non_es_fields = ['id', 'name__localized_string'] + list(fields)
if q.isnumeric():
qs = (Webapp.objects.filter(pk=q)
.values(*non_es_fields))
else:
# Try to load by GUID:
qs = (Webapp.objects.filter(guid=q)
.values(*non_es_fields))
if not qs.count():
qs = (S(WebappIndexer)
.query(should=True, **_expand_query(q, fields))
.values_dict(*['id'] + list(fields)))
qs = _slice_results(request, qs)
for app in qs:
if 'name__localized_string' in app:
# This is a result from the database.
app['url'] = reverse('lookup.app_summary', args=[app['id']])
app['name'] = app['name__localized_string']
results.append(app)
else:
# This is a result from elasticsearch which returns name as a list.
app['url'] = reverse('lookup.app_summary', args=[app['id']])
for field in ('id', 'app_slug'):
app[field] = app.get(field)
for name in app['name']:
dd = app.copy()
dd['name'] = name
results.append(dd)
return {'results': results}
def get_doctype_stats(index):
"""Returns a dict of name -> count for documents indexed.
For example:
>>> get_doctype_stats()
{'questions_question': 14216, 'forums_thread': 419, 'wiki_document': 759}
:throws elasticsearch.exceptions.ConnectionError: if there is a
connection error, including a timeout.
:throws elasticsearch.exceptions.NotFound: if the index doesn't exist
"""
stats = {}
from kitsune.search.models import get_mapping_types
for cls in get_mapping_types():
if cls.get_index() == index:
# Note: Can't use cls.search() here since that returns a
# Sphilastic which is hard-coded to look only at the
# read index..
s = S(cls).indexes(index)
stats[cls.get_mapping_type_name()] = s.count()
return stats
def get_index_stats():
"""Return dict of name -> count for documents indexed.
For example:
>>> get_index_stats()
{'response': 122233}
.. Note::
This infers the index to use from the registered mapping
types.
:returns: mapping type name -> count for documents indexes.
:throws pyelasticsearch.exceptions.Timeout: if the request
times out
:throws pyelasticsearch.exceptions.ConnectionError: if there's a
connection error
:throws pyelasticsearch.exceptions.ElasticHttpNotFound: if the
index doesn't exist
"""
stats = {}
for name, cls in get_mapping_types().items():
stats[name] = S(cls).count()
return stats
def search(cls):
"""Returns a typed S for this class.
:returns: an `S`
"""
return S(cls)
def app_search(request):
results = []
q = request.GET.get('q', u'').lower().strip()
addon_type = request.GET.get('type', amo.ADDON_WEBAPP)
fields = ('name', 'app_slug')
non_es_fields = ['id', 'name__localized_string'] + list(fields)
if q.isnumeric():
qs = (Addon.objects.filter(type=addon_type,
pk=q).values(*non_es_fields))
else:
# Try to load by GUID:
qs = (Addon.objects.filter(type=addon_type,
guid=q).values(*non_es_fields))
if not qs.count():
qs = (S(Addon).query(type=addon_type, or_=_expand_query(
q, fields)).values_dict(*fields)[:20])
for app in qs:
app['url'] = reverse('lookup.app_summary', args=[app['id']])
# ES returns a list of localized names but database queries do not.
if type(app['name']) != list:
app['name'] = [app['name__localized_string']]
for name in app['name']:
dd = app.copy()
dd['name'] = name
results.append(dd)
return {'results': results}
def test_facet_raw_overrides_facet(self):
"""facet_raw overrides facet with the same facet name."""
qs = (S(FakeDjangoMappingType)
.query(foo='car')
.facet('tag')
.facet_raw(tag={'terms': {'field': 'tag'}, 'global': True}))
eq_(facet_counts_dict(qs, 'tag'),
dict(awesome=3, boring=1, boat=1))
def test_global_facet(self):
qs = S(FakeDjangoMappingType).query(foo='car').filter(width=5)
# facet restricted to query
eq_(facet_counts_dict(qs.facet('tag'), 'tag'), {'awesome': 2})
# facet applies to all of corpus
eq_(facet_counts_dict(qs.facet('tag', global_=True), 'tag'),
dict(awesome=3, boring=1, boat=1))
def test_filtered_facet(self):
qs = S(FakeDjangoMappingType).query(foo='car').filter(width=5)
# filter doesn't apply to facets
eq_(facet_counts_dict(qs.facet('tag'), 'tag'), {'awesome': 2})
# filter does apply to facets
eq_(facet_counts_dict(qs.facet('tag', filtered=True), 'tag'),
{'awesome': 1})
def search(cls, query, vouched=None, photo=None):
"""Sensible default search for UserProfiles."""
query = query.lower().strip()
fields = ('username', 'bio__text', 'website', 'email', 'groups',
'skills', 'languages', 'first_name__prefix',
'last_name__prefix', 'ircname', 'country', 'region', 'city')
if query:
q = dict((field, query) for field in fields)
s = S(cls).query(or_=q)
else:
s = S(cls)
if vouched is not None:
s = s.filter(is_vouched=vouched)
if photo is not None:
s = s.filter(has_photo=photo)
return s
def test_index(self):
document = {'id': 1, 'name': 'odin skullcrusher'}
# Generate the FakeModel in our "database"
FakeModel(**document)
# Index the document with .index()
FakeDjangoMappingType.index(document, id_=document['id'])
IndexableTest.refresh()
# Query it to make sure it's there.
eq_(len(S(FakeDjangoMappingType).query(name__prefix='odin')), 1)
def test_index(self):
self.persist_data([
{
'id': 1,
'name': 'odin skullcrusher'
},
{
'id': 2,
'name': 'olaf bloodbiter'
},
])
# Query it to make sure it's there.
eq_(len(S(FakeDjangoMappingType).query(name__prefix='odin')), 1)
def test_bulk_index(self):
documents = [{
'id': 1,
'name': 'odin skullcrusher'
}, {
'id': 2,
'name': 'heimdall kneebiter'
}, {
'id': 3,
'name': 'erik rose'
}]
# Generate the FakeModel in our "database"
for doc in documents:
FakeModel(**doc)
# Index the document with .index()
FakeDjangoMappingType.bulk_index(documents, id_field='id')
IndexableTest.refresh()
# Query it to make sure they're there.
eq_(len(S(FakeDjangoMappingType).query(name__prefix='odin')), 1)
eq_(len(S(FakeDjangoMappingType).query(name__prefix='erik')), 1)
def test_get_object(self):
self.persist_data([
{
'id': 1,
'name': 'odin skullcrusher'
},
{
'id': 2,
'name': 'olaf bloodbiter'
},
])
s = S(FakeDjangoMappingType).query(name__prefix='odin')
obj = s[0]
eq_(obj.object.id, 1)
def featured_suggestions(request):
q = request.GET.get('q', u'').lower().strip()
cat_slug = request.GET.get('category')
filters = {
'type': amo.ADDON_WEBAPP,
'status': amo.STATUS_PUBLIC,
'is_disabled': False,
}
if cat_slug:
filters.update({'category': cat_slug})
search_fields = ['app_slug', 'name']
# If search looks like an ID, also search the ID field.
if q.isdigit():
qs = search_fields.append('id')
# Do a search based on the query string.
qs = S(WebappIndexer).filter(**filters).query(should=True,
**dict(
('{0}__prefix'.format(f),
w)
for f in search_fields
for w in q.split()))
fields = ['id', 'app_slug', 'default_locale']
for analyzer in amo.SEARCH_ANALYZER_MAP:
if (not settings.ES_USE_PLUGINS
and analyzer in amo.SEARCH_ANALYZER_PLUGINS):
continue
fields.append('name_{0}'.format(analyzer))
qs = qs.values_dict(*fields)
results = []
for app in qs[:20]:
results.append({
'id':
app._id,
'name':
get_attr_lang(app, 'name', app['default_locale']),
'url':
reverse('detail', args=[app['app_slug']]),
})
return results
def from_search(cls, cat=None, region=None, gaia=False):
filters = dict(type=amo.ADDON_WEBAPP,
status=amo.STATUS_PUBLIC,
is_disabled=False)
if cat:
filters.update(category=cat.id)
srch = S(cls).query(**filters)
if region:
excluded = cls.get_excluded_in(region)
if excluded:
srch = srch.filter(~F(id__in=excluded))
if waffle.switch_is_active('disabled-payments') or not gaia:
srch = srch.filter(premium_type__in=amo.ADDON_FREES, price=0)
return srch
def from_search(cls,
cat=None,
region=None,
gaia=False,
mobile=False,
tablet=False,
filter_overrides=None):
filters = dict(type=amo.ADDON_WEBAPP,
status=amo.STATUS_PUBLIC,
is_disabled=False)
# Special handling if status is 'any' to remove status filter.
if filter_overrides and 'status' in filter_overrides:
if filter_overrides['status'] is 'any':
del filters['status']
del filter_overrides['status']
if filter_overrides:
filters.update(filter_overrides)
if cat:
filters.update(category=cat.id)
srch = S(cls).query(**filters)
if region:
excluded = cls.get_excluded_in(region)
if excluded:
srch = srch.filter(~F(id__in=excluded))
if mobile:
srch = srch.filter(uses_flash=False)
if (mobile or tablet) and not gaia:
# Don't show packaged apps on Firefox for Android.
srch = srch.filter(app_type=amo.ADDON_WEBAPP_HOSTED)
# Only show premium apps on gaia and desktop for now.
srch = srch.filter(
~F(premium_type__in=amo.ADDON_PREMIUMS, price__gt=0))
return srch
def apply_filters(self, request, applicable_filters):
"""Implement advanced filters.
- Implement 'groups' filter.
- Implement 'languages' filter.
- Implement 'skills' filter.
"""
if (request.GET.get('restricted', False)
and 'email__text' not in applicable_filters
and len(applicable_filters) != 1):
raise ImmediateHttpResponse(response=http.HttpForbidden())
if request.GET.get('restricted', False):
applicable_filters.append(F(allows_community_sites=True))
mega_filter = F()
for filter in applicable_filters:
mega_filter &= filter
return S(UserProfile).filter(mega_filter)
def queryset(self):
"""Get items based on ID or search by name."""
results = Addon.objects.none()
q = self.request.GET.get(self.key)
if q:
pk = None
try:
pk = int(q)
except ValueError:
pass
qs = None
if pk:
qs = Addon.objects.filter(id=int(q), disabled_by_user=False)
elif len(q) > 2:
# Oh, how I wish I could elastically exclude terms.
# (You can now, but I forgot why I was complaining to
# begin with.)
qs = (S(Addon).query(or_=name_only_query(q.lower())).filter(
is_disabled=False))
if qs:
results = qs.filter(type__in=self.types,
status__in=amo.REVIEWED_STATUSES)
return results
def test_filter_bad_field_action(self):
with self.assertRaises(InvalidFieldActionError):
len(S(FakeDjangoMappingType).filter(F(tag__faux='awesome')))
def test_order_by(self):
res = S(FakeDjangoMappingType).filter(tag='awesome').order_by('-width')
eq_([d.id for d in res], [5, 3, 1])
def get_obj(self):
return S(WebappIndexer).filter(id=self.app.pk).execute().objects[0]
def test_facet(self):
qs = S(FakeDjangoMappingType).facet('tag')
eq_(facet_counts_dict(qs, 'tag'), dict(awesome=3, boring=1, boat=1))
