def test_filter_or(self):
qs = Addon.search().filter(type=1).filter(or_=dict(status=1, app=2))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [
# {'term': {'type': 1}},
# {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]},
# ]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
or_clause = sorted(filters['and'])[0]
assert or_clause.keys() == ['or']
assert {'term': {'status': 1}} in or_clause['or']
assert {'term': {'app': 2}} in or_clause['or']
qs = Addon.search().filter(type=1, or_=dict(status=1, app=2))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [
# {'term': {'type': 1}},
# {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]},
# ]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
or_clause = sorted(filters['and'])[0]
assert or_clause.keys() == ['or']
assert {'term': {'status': 1}} in or_clause['or']
assert {'term': {'app': 2}} in or_clause['or']
def test_extra_order_by(self):
qs = Addon.search().extra(order_by=['-rating'])
assert qs._build_query()['sort'] == [{'rating': 'desc'}]
qs = Addon.search().order_by('-id').extra(order_by=['-rating'])
assert qs._build_query()['sort'] == [
{'id': 'desc'}, {'rating': 'desc'}]
def test_count_non_dsl_mode(self):
p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False)
assert p._count is None
p.page(1)
assert p.count == Addon.search().count()
def test_extra_query(self):
qs = Addon.search().extra(query={'type': 1})
eq_(qs._build_query()['query']['function_score']['query'],
{'term': {'type': 1}})
qs = Addon.search().filter(status=1).extra(query={'type': 1})
filtered = qs._build_query()['query']['filtered']
eq_(filtered['query']['function_score']['query'],
{'term': {'type': 1}})
eq_(filtered['filter'], [{'term': {'status': 1}}])
def test_extra_query(self):
qs = Addon.search().extra(query={'type': 1})
assert qs._build_query()['query'] == (
{'term': {'type': 1}})
qs = Addon.search().filter(status=1).extra(query={'type': 1})
filtered = qs._build_query()['query']['bool']
assert filtered['must'] == (
[{'term': {'type': 1}}])
assert filtered['filter'] == [{'term': {'status': 1}}]
def test_extra_filter(self):
qs = Addon.search().extra(filter={'category__in': [1, 2]})
assert qs._build_query()['query']['bool']['filter'] == (
[{'terms': {'category': [1, 2]}}])
qs = (Addon.search().filter(type=1)
.extra(filter={'category__in': [1, 2]}))
filters = qs._build_query()['query']['bool']['filter']
# Filters:
# [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}]
assert len(filters) == 2
assert {'term': {'type': 1}} in filters
assert {'terms': {'category': [1, 2]}} in filters
def test_extra_filter(self):
qs = Addon.search().extra(filter={'category__in': [1, 2]})
assert qs._build_query()['query']['filtered']['filter'] == (
[{'in': {'category': [1, 2]}}])
qs = (Addon.search().filter(type=1)
.extra(filter={'category__in': [1, 2]}))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
assert {'in': {'category': [1, 2]}} in filters['and']
def test_extra_filter(self):
qs = Addon.search().extra(filter={'category__in': [1, 2]})
eq_(qs._build_query()['query']['filtered']['filter'],
[{'in': {'category': [1, 2]}}])
qs = (Addon.search().filter(type=1)
.extra(filter={'category__in': [1, 2]}))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]}
eq_(filters.keys(), ['and'])
ok_({'term': {'type': 1}} in filters['and'])
ok_({'in': {'category': [1, 2]}} in filters['and'])
def test_count_non_dsl_mode(self):
addon_factory()
addon_factory()
addon_factory()
self.refresh()
p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False)
assert p.count == 3
p.page(1)
assert p.count == 3
assert p.count == Addon.search().count()
def es_extensions(request, category=None, template=None):
TYPE = amo.ADDON_EXTENSION
if category is not None:
q = Category.objects.filter(application=request.APP.id, type=TYPE)
category = get_object_or_404(q, slug=category)
if ('sort' not in request.GET and not request.MOBILE
and category and category.count > 4):
return category_landing(request, category)
qs = (Addon.search().filter(type=TYPE, app=request.APP.id,
is_disabled=False,
status__in=amo.REVIEWED_STATUSES))
filter = ESAddonFilter(request, qs, key='sort', default='popular')
qs, sorting = filter.qs, filter.field
src = 'cb-btn-%s' % sorting
dl_src = 'cb-dl-%s' % sorting
if category:
qs = qs.filter(category=category.id)
addons = amo.utils.paginate(request, qs)
return render(request, template,
{'section': 'extensions', 'addon_type': TYPE,
'category': category, 'addons': addons,
'filter': filter, 'sorting': sorting,
'sort_opts': filter.opts, 'src': src,
'dl_src': dl_src, 'search_cat': '%s,0' % TYPE})
def test_indexed_count(self):
# Did all the right addons get indexed?
count = Addon.search().filter(type=1, is_disabled=False).count()
# Created in the setUpClass.
assert count == 4 == (
Addon.objects.filter(disabled_by_user=False,
status__in=amo.VALID_ADDON_STATUSES).count())
def test_facet_range(self):
facet = {'range': {'status': [{'lte': 3}, {'gte': 5}]}}
# Pass a copy so edits aren't propagated back here.
qs = Addon.search().filter(app=1).facet(by_status=dict(facet))
assert qs._build_query()['query']['filtered']['filter'] == (
[{'term': {'app': 1}}])
assert qs._build_query()['facets'] == {'by_status': facet}
def test_validate_number(self):
p = amo.utils.ESPaginator(Addon.search(), 20)
# A bad number raises an exception.
with self.assertRaises(paginator.PageNotAnInteger):
p.page('a')
# A large number is ignored.
p.page(99)
def test_and(self):
qs = Addon.search().filter(type=1, category__in=[1, 2])
filters = qs._build_query()['query']['bool']['filter']
# Filters:
# [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}]
assert len(filters) == 2
assert {'term': {'type': 1}} in filters
assert {'terms': {'category': [1, 2]}} in filters
def test_and(self):
qs = Addon.search().filter(type=1, category__in=[1, 2])
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}]}
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
assert {'in': {'category': [1, 2]}} in filters['and']
def test_paginate_returns_this_paginator(self):
request = MagicMock()
request.GET.get.return_value = 1
request.GET.urlencode.return_value = ''
request.path = ''
qs = Addon.search()
pager = paginate(request, qs)
assert isinstance(pager.paginator, ESPaginator)
def test_query_multiple_and_range(self):
qs = Addon.search().query(type=1, status__gte=1)
query = qs._build_query()['query']['function_score']['query']
# Query:
# {'bool': {'must': [{'term': {'type': 1}},
# {'range': {'status': {'gte': 1}}}, ]}}
eq_(query.keys(), ['bool'])
eq_(query['bool'].keys(), ['must'])
ok_({'term': {'type': 1}} in query['bool']['must'])
ok_({'range': {'status': {'gte': 1}}} in query['bool']['must'])
def test_query_multiple_and_range(self):
qs = Addon.search().query(type=1, status__gte=1)
query = qs._build_query()['query']
# Query:
# {'bool': {'must': [{'term': {'type': 1}},
# {'range': {'status': {'gte': 1}}}, ]}}
assert query.keys() == ['bool']
assert query['bool'].keys() == ['must']
assert {'term': {'type': 1}} in query['bool']['must']
assert {'range': {'status': {'gte': 1}}} in query['bool']['must']
def test_query_or(self):
qs = Addon.search().query(or_=dict(type=1, status__gte=2))
query = qs._build_query()['query']['function_score']['query']
# Query:
# {'bool': {'should': [{'term': {'type': 1}},
# {'range': {'status': {'gte': 2}}}, ]}}
assert query.keys() == ['bool']
assert query['bool'].keys() == ['should']
assert {'term': {'type': 1}} in query['bool']['should']
assert {'range': {'status': {'gte': 2}}} in query['bool']['should']
def test_gt(self):
qs = Addon.search().filter(type__in=[1, 2], status__gt=4)
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [
# {'in': {'type': [1, 2]}},
# {'range': {'status': {'gt': 4}}},
# ]}
eq_(filters.keys(), ['and'])
ok_({'in': {'type': [1, 2]}} in filters['and'])
ok_({'range': {'status': {'gt': 4}}} in filters['and'])
def test_query_fuzzy(self):
fuzz = {'boost': 2, 'value': 'woo'}
qs = Addon.search().query(or_=dict(type=1, status__fuzzy=fuzz))
query = qs._build_query()['query']['function_score']['query']
# Query:
# {'bool': {'should': [{'fuzzy': {'status': fuzz}},
# {'term': {'type': 1}}, ]}})
eq_(query.keys(), ['bool'])
eq_(query['bool'].keys(), ['should'])
ok_({'term': {'type': 1}} in query['bool']['should'])
ok_({'fuzzy': {'status': fuzz}} in query['bool']['should'])
def test_query_fuzzy(self):
fuzz = {'boost': 2, 'value': 'woo'}
qs = Addon.search().query(type=1, status__fuzzy=fuzz)
query = qs._build_query()['query']
# Query:
# {'bool': {'must': [{'fuzzy': {'status': fuzz}},
# {'term': {'type': 1}}, ]}})
assert query.keys() == ['bool']
assert query['bool'].keys() == ['must']
assert {'term': {'type': 1}} in query['bool']['must']
assert {'fuzzy': {'status': fuzz}} in query['bool']['must']
def test_lt(self):
qs = Addon.search().filter(type__in=[1, 2], status__lt=4)
filters = qs._build_query()['query']['bool']['filter']
# Filters:
# [
# {'range': {'status': {'lt': 4}}},
# {'terms': {'type': [1, 2]}},
# ]
assert len(filters)
assert {'range': {'status': {'lt': 4}}} in filters
assert {'terms': {'type': [1, 2]}} in filters
def test_lt(self):
qs = Addon.search().filter(type__in=[1, 2], status__lt=4)
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [
# {'range': {'status': {'lt': 4}}},
# {'in': {'type': [1, 2]}},
# ]}
assert filters.keys() == ['and']
assert {'range': {'status': {'lt': 4}}} in filters['and']
assert {'in': {'type': [1, 2]}} in filters['and']
def addon_search(request):
ctx = {}
if 'q' in request.GET:
q = ctx['q'] = request.GET['q']
if q.isdigit():
qs = Addon.objects.filter(id=int(q))
else:
qs = Addon.search().query(name__text=q.lower())[:100]
if len(qs) == 1:
return redirect('zadmin.addon_manage', qs[0].id)
ctx['addons'] = qs
return render(request, 'zadmin/addon-search.html', ctx)
def test_extra_filter_or(self):
qs = Addon.search().extra(filter={'or_': {'status': 1, 'app': 2}})
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# [{'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}])
eq_(len(filters), 1)
eq_(filters[0].keys(), ['or'])
ok_({'term': {'status': 1}} in filters[0]['or'])
ok_({'term': {'app': 2}} in filters[0]['or'])
qs = (Addon.search().filter(type=1)
.extra(filter={'or_': {'status': 1, 'app': 2}}))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}},
# {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}]})
eq_(filters.keys(), ['and'])
ok_({'term': {'type': 1}} in filters['and'])
or_clause = sorted(filters['and'])[0]
eq_(or_clause.keys(), ['or'])
ok_({'term': {'status': 1}} in or_clause['or'])
ok_({'term': {'app': 2}} in or_clause['or'])
def test_extra_filter_or(self):
qs = Addon.search().extra(filter={'or_': {'status': 1, 'app': 2}})
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# [{'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}])
assert len(filters) == 1
assert filters[0].keys() == ['or']
assert {'term': {'status': 1}} in filters[0]['or']
assert {'term': {'app': 2}} in filters[0]['or']
qs = (Addon.search().filter(type=1)
.extra(filter={'or_': {'status': 1, 'app': 2}}))
filters = qs._build_query()['query']['filtered']['filter']
# Filters:
# {'and': [{'term': {'type': 1}},
# {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}]})
assert filters.keys() == ['and']
assert {'term': {'type': 1}} in filters['and']
or_clause = sorted(filters['and'])[0]
assert or_clause.keys() == ['or']
assert {'term': {'status': 1}} in or_clause['or']
assert {'term': {'app': 2}} in or_clause['or']
def test_query_or_and(self):
qs = Addon.search().query(or_=dict(type=1, status__gte=2), category=2)
query = qs._build_query()['query']['function_score']['query']
# Query:
# {'bool': {'must': [{'term': {'category': 2}},
# {'bool': {'should': [
# {'term': {'type': 1}},
# {'range': {'status': {'gte': 2}}}, ]}}]}})
eq_(query.keys(), ['bool'])
eq_(query['bool'].keys(), ['must'])
ok_({'term': {'category': 2}} in query['bool']['must'])
sub_clause = sorted(query['bool']['must'])[0]
eq_(sub_clause.keys(), ['bool'])
eq_(sub_clause['bool'].keys(), ['should'])
ok_({'range': {'status': {'gte': 2}}} in sub_clause['bool']['should'])
ok_({'term': {'type': 1}} in sub_clause['bool']['should'])
def test_aggregations(self):
Tag(tag_text='sky').save_tag(self._addons[0])
Tag(tag_text='sky').save_tag(self._addons[1])
Tag(tag_text='sky').save_tag(self._addons[2])
Tag(tag_text='earth').save_tag(self._addons[0])
Tag(tag_text='earth').save_tag(self._addons[1])
Tag(tag_text='ocean').save_tag(self._addons[0])
self.reindex(Addon)
qs = Addon.search().aggregate(tags={'terms': {'field': 'tags'}})
results = list(qs)
assert len(results) == 6
assert qs.aggregations == {
u'tags': [
{u'doc_count': 3, u'key': u'sky'},
{u'doc_count': 2, u'key': u'earth'},
{u'doc_count': 1, u'key': u'ocean'}]}
def themes_search(request):
search_form = forms.ThemeSearchForm(request.GET)
if search_form.is_valid():
q = search_form.cleaned_data['q']
rereview = search_form.cleaned_data['queue_type'] == 'rereview'
flagged = search_form.cleaned_data['queue_type'] == 'flagged'
# ES query on name.
themes = Addon.search().filter(type=amo.ADDON_PERSONA)
if rereview:
themes = themes.filter(has_theme_rereview=True)
else:
themes = themes.filter(status=(amo.STATUS_REVIEW_PENDING if flagged
else amo.STATUS_PENDING),
has_theme_rereview=False)
themes = themes.filter_query_string(q)[:100]
now = datetime.datetime.now()
reviewers = []
for theme in themes:
try:
themelock = theme.persona.themelock
if themelock.expiry > now:
reviewers.append(themelock.reviewer.email)
else:
reviewers.append('')
except ObjectDoesNotExist:
reviewers.append('')
themes = list(themes.values_dict('name', 'slug', 'status'))
for theme, reviewer in zip(themes, reviewers):
# Collapse single value fields from a list.
theme['id'] = theme['id']
theme['slug'] = theme['slug']
theme['status'] = theme['status']
# Dehydrate.
theme['reviewer'] = reviewer
return {'objects': themes, 'meta': {'total_count': len(themes)}}
def test_getitem(self):
addons = list(Addon.search())
eq_(addons[0], Addon.search()[0])
def test_iter(self):
qs = Addon.search().filter(type=1, is_disabled=False)
eq_(len(qs), len(list(qs)))
def test_count(self):
eq_(Addon.search().count(), 6)
def test_count_uses_cached_results(self):
qs = Addon.search()
qs._results_cache = mock.Mock()
qs._results_cache.count = mock.sentinel.count
eq_(qs.count(), mock.sentinel.count)
def test_len(self):
qs = Addon.search()
qs._results_cache = [1]
eq_(len(qs), 1)
def test_not_indexed(self):
addon = Addon.objects.create(type=amo.ADDON_EXTENSION,
status=amo.STATUS_PUBLIC)
assert issubclass(
Addon.search().filter(id__in=addon.id).count().__class__,
mock.Mock)
def test_slice_stop(self):
qs = Addon.search()[:6]
eq_(qs._build_query()['size'], 6)
def test_order_by_multiple(self):
qs = Addon.search().order_by('-rating', 'id')
eq_(qs._build_query()['sort'], [{'rating': 'desc'}, 'id'])
def test_values_dict_result(self):
addons = [{'id': [a.id], 'slug': [a.slug]} for a in self._addons]
qs = Addon.search().values_dict('slug').order_by('id')
eq_(list(qs), list(addons))
def test_slice_stop_zero(self):
qs = Addon.search()[:0]
eq_(qs._build_query()['size'], 0)
def test_empty_values_dict_result(self):
qs = Addon.search().values_dict()
# Look for some of the keys we expect.
for key in ('id', 'name', 'status', 'app'):
assert key in qs[0].keys(), qs[0].keys()
def test_object_result(self):
qs = Addon.search().filter(id=self._addons[0].id)[:1]
eq_(self._addons[:1], list(qs))
def test_object_result_slice(self):
addon = self._addons[0]
qs = Addon.search().filter(id=addon.id)
eq_(addon, qs[0])
def test_extra_bad_key(self):
with self.assertRaises(AssertionError):
Addon.search().extra(x=1)
def test_extra_values_dict(self):
qs = Addon.search().extra(values_dict=['name'])
eq_(qs._build_query()['fields'], ['id', 'name'])
qs = Addon.search().values_dict('status').extra(values_dict=['name'])
eq_(qs._build_query()['fields'], ['id', 'status', 'name'])
def test_empty_values_dict(self):
qs = Addon.search().values_dict()
assert 'fields' not in qs._build_query()
def test_values_dict(self):
qs = Addon.search().values_dict('name')
eq_(qs._build_query()['fields'], ['id', 'name'])
def test_slice(self):
qs = Addon.search()[5:12]
eq_(qs._build_query()['from'], 5)
eq_(qs._build_query()['size'], 7)
def test_prefix(self):
qs = Addon.search().query(name__startswith='woo')
eq_(qs._build_query()['query']['function_score']['query'],
{'prefix': {
'name': 'woo'
}})
def test_order_by_desc(self):
qs = Addon.search().order_by('-rating')
eq_(qs._build_query()['sort'], [{'rating': 'desc'}])
def test_order_by_asc(self):
qs = Addon.search().order_by('rating')
eq_(qs._build_query()['sort'], ['rating'])
