def test_filter_or(self): qs = Addon.search().filter(type=1).filter(or_=dict(status=1, app=2)) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [ # {'term': {'type': 1}}, # {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}, # ]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] or_clause = sorted(filters['and'])[0] assert or_clause.keys() == ['or'] assert {'term': {'status': 1}} in or_clause['or'] assert {'term': {'app': 2}} in or_clause['or'] qs = Addon.search().filter(type=1, or_=dict(status=1, app=2)) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [ # {'term': {'type': 1}}, # {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}, # ]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] or_clause = sorted(filters['and'])[0] assert or_clause.keys() == ['or'] assert {'term': {'status': 1}} in or_clause['or'] assert {'term': {'app': 2}} in or_clause['or']
def test_extra_order_by(self): qs = Addon.search().extra(order_by=['-rating']) assert qs._build_query()['sort'] == [{'rating': 'desc'}] qs = Addon.search().order_by('-id').extra(order_by=['-rating']) assert qs._build_query()['sort'] == [ {'id': 'desc'}, {'rating': 'desc'}]
def test_count_non_dsl_mode(self): p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False) assert p._count is None p.page(1) assert p.count == Addon.search().count()
def test_extra_query(self): qs = Addon.search().extra(query={'type': 1}) eq_(qs._build_query()['query']['function_score']['query'], {'term': {'type': 1}}) qs = Addon.search().filter(status=1).extra(query={'type': 1}) filtered = qs._build_query()['query']['filtered'] eq_(filtered['query']['function_score']['query'], {'term': {'type': 1}}) eq_(filtered['filter'], [{'term': {'status': 1}}])
def test_extra_query(self): qs = Addon.search().extra(query={'type': 1}) assert qs._build_query()['query'] == ( {'term': {'type': 1}}) qs = Addon.search().filter(status=1).extra(query={'type': 1}) filtered = qs._build_query()['query']['bool'] assert filtered['must'] == ( [{'term': {'type': 1}}]) assert filtered['filter'] == [{'term': {'status': 1}}]
def test_extra_filter(self): qs = Addon.search().extra(filter={'category__in': [1, 2]}) assert qs._build_query()['query']['bool']['filter'] == ( [{'terms': {'category': [1, 2]}}]) qs = (Addon.search().filter(type=1) .extra(filter={'category__in': [1, 2]})) filters = qs._build_query()['query']['bool']['filter'] # Filters: # [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}] assert len(filters) == 2 assert {'term': {'type': 1}} in filters assert {'terms': {'category': [1, 2]}} in filters
def test_extra_filter(self): qs = Addon.search().extra(filter={'category__in': [1, 2]}) assert qs._build_query()['query']['filtered']['filter'] == ( [{'in': {'category': [1, 2]}}]) qs = (Addon.search().filter(type=1) .extra(filter={'category__in': [1, 2]})) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] assert {'in': {'category': [1, 2]}} in filters['and']
def test_extra_filter(self): qs = Addon.search().extra(filter={'category__in': [1, 2]}) eq_(qs._build_query()['query']['filtered']['filter'], [{'in': {'category': [1, 2]}}]) qs = (Addon.search().filter(type=1) .extra(filter={'category__in': [1, 2]})) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}, ]} eq_(filters.keys(), ['and']) ok_({'term': {'type': 1}} in filters['and']) ok_({'in': {'category': [1, 2]}} in filters['and'])
def test_count_non_dsl_mode(self): addon_factory() addon_factory() addon_factory() self.refresh() p = ESPaginator(Addon.search(), 20, use_elasticsearch_dsl=False) assert p.count == 3 p.page(1) assert p.count == 3 assert p.count == Addon.search().count()
def es_extensions(request, category=None, template=None): TYPE = amo.ADDON_EXTENSION if category is not None: q = Category.objects.filter(application=request.APP.id, type=TYPE) category = get_object_or_404(q, slug=category) if ('sort' not in request.GET and not request.MOBILE and category and category.count > 4): return category_landing(request, category) qs = (Addon.search().filter(type=TYPE, app=request.APP.id, is_disabled=False, status__in=amo.REVIEWED_STATUSES)) filter = ESAddonFilter(request, qs, key='sort', default='popular') qs, sorting = filter.qs, filter.field src = 'cb-btn-%s' % sorting dl_src = 'cb-dl-%s' % sorting if category: qs = qs.filter(category=category.id) addons = amo.utils.paginate(request, qs) return render(request, template, {'section': 'extensions', 'addon_type': TYPE, 'category': category, 'addons': addons, 'filter': filter, 'sorting': sorting, 'sort_opts': filter.opts, 'src': src, 'dl_src': dl_src, 'search_cat': '%s,0' % TYPE})
def test_indexed_count(self): # Did all the right addons get indexed? count = Addon.search().filter(type=1, is_disabled=False).count() # Created in the setUpClass. assert count == 4 == ( Addon.objects.filter(disabled_by_user=False, status__in=amo.VALID_ADDON_STATUSES).count())
def test_facet_range(self): facet = {'range': {'status': [{'lte': 3}, {'gte': 5}]}} # Pass a copy so edits aren't propagated back here. qs = Addon.search().filter(app=1).facet(by_status=dict(facet)) assert qs._build_query()['query']['filtered']['filter'] == ( [{'term': {'app': 1}}]) assert qs._build_query()['facets'] == {'by_status': facet}
def test_validate_number(self): p = amo.utils.ESPaginator(Addon.search(), 20) # A bad number raises an exception. with self.assertRaises(paginator.PageNotAnInteger): p.page('a') # A large number is ignored. p.page(99)
def test_and(self): qs = Addon.search().filter(type=1, category__in=[1, 2]) filters = qs._build_query()['query']['bool']['filter'] # Filters: # [{'term': {'type': 1}}, {'terms': {'category': [1, 2]}}] assert len(filters) == 2 assert {'term': {'type': 1}} in filters assert {'terms': {'category': [1, 2]}} in filters
def test_and(self): qs = Addon.search().filter(type=1, category__in=[1, 2]) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, {'in': {'category': [1, 2]}}]} assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] assert {'in': {'category': [1, 2]}} in filters['and']
def test_paginate_returns_this_paginator(self): request = MagicMock() request.GET.get.return_value = 1 request.GET.urlencode.return_value = '' request.path = '' qs = Addon.search() pager = paginate(request, qs) assert isinstance(pager.paginator, ESPaginator)
def test_query_multiple_and_range(self): qs = Addon.search().query(type=1, status__gte=1) query = qs._build_query()['query']['function_score']['query'] # Query: # {'bool': {'must': [{'term': {'type': 1}}, # {'range': {'status': {'gte': 1}}}, ]}} eq_(query.keys(), ['bool']) eq_(query['bool'].keys(), ['must']) ok_({'term': {'type': 1}} in query['bool']['must']) ok_({'range': {'status': {'gte': 1}}} in query['bool']['must'])
def test_query_multiple_and_range(self): qs = Addon.search().query(type=1, status__gte=1) query = qs._build_query()['query'] # Query: # {'bool': {'must': [{'term': {'type': 1}}, # {'range': {'status': {'gte': 1}}}, ]}} assert query.keys() == ['bool'] assert query['bool'].keys() == ['must'] assert {'term': {'type': 1}} in query['bool']['must'] assert {'range': {'status': {'gte': 1}}} in query['bool']['must']
def test_query_or(self): qs = Addon.search().query(or_=dict(type=1, status__gte=2)) query = qs._build_query()['query']['function_score']['query'] # Query: # {'bool': {'should': [{'term': {'type': 1}}, # {'range': {'status': {'gte': 2}}}, ]}} assert query.keys() == ['bool'] assert query['bool'].keys() == ['should'] assert {'term': {'type': 1}} in query['bool']['should'] assert {'range': {'status': {'gte': 2}}} in query['bool']['should']
def test_gt(self): qs = Addon.search().filter(type__in=[1, 2], status__gt=4) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [ # {'in': {'type': [1, 2]}}, # {'range': {'status': {'gt': 4}}}, # ]} eq_(filters.keys(), ['and']) ok_({'in': {'type': [1, 2]}} in filters['and']) ok_({'range': {'status': {'gt': 4}}} in filters['and'])
def test_query_fuzzy(self): fuzz = {'boost': 2, 'value': 'woo'} qs = Addon.search().query(or_=dict(type=1, status__fuzzy=fuzz)) query = qs._build_query()['query']['function_score']['query'] # Query: # {'bool': {'should': [{'fuzzy': {'status': fuzz}}, # {'term': {'type': 1}}, ]}}) eq_(query.keys(), ['bool']) eq_(query['bool'].keys(), ['should']) ok_({'term': {'type': 1}} in query['bool']['should']) ok_({'fuzzy': {'status': fuzz}} in query['bool']['should'])
def test_query_fuzzy(self): fuzz = {'boost': 2, 'value': 'woo'} qs = Addon.search().query(type=1, status__fuzzy=fuzz) query = qs._build_query()['query'] # Query: # {'bool': {'must': [{'fuzzy': {'status': fuzz}}, # {'term': {'type': 1}}, ]}}) assert query.keys() == ['bool'] assert query['bool'].keys() == ['must'] assert {'term': {'type': 1}} in query['bool']['must'] assert {'fuzzy': {'status': fuzz}} in query['bool']['must']
def test_lt(self): qs = Addon.search().filter(type__in=[1, 2], status__lt=4) filters = qs._build_query()['query']['bool']['filter'] # Filters: # [ # {'range': {'status': {'lt': 4}}}, # {'terms': {'type': [1, 2]}}, # ] assert len(filters) assert {'range': {'status': {'lt': 4}}} in filters assert {'terms': {'type': [1, 2]}} in filters
def test_lt(self): qs = Addon.search().filter(type__in=[1, 2], status__lt=4) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [ # {'range': {'status': {'lt': 4}}}, # {'in': {'type': [1, 2]}}, # ]} assert filters.keys() == ['and'] assert {'range': {'status': {'lt': 4}}} in filters['and'] assert {'in': {'type': [1, 2]}} in filters['and']
def addon_search(request): ctx = {} if 'q' in request.GET: q = ctx['q'] = request.GET['q'] if q.isdigit(): qs = Addon.objects.filter(id=int(q)) else: qs = Addon.search().query(name__text=q.lower())[:100] if len(qs) == 1: return redirect('zadmin.addon_manage', qs[0].id) ctx['addons'] = qs return render(request, 'zadmin/addon-search.html', ctx)
def test_extra_filter_or(self): qs = Addon.search().extra(filter={'or_': {'status': 1, 'app': 2}}) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # [{'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}]) eq_(len(filters), 1) eq_(filters[0].keys(), ['or']) ok_({'term': {'status': 1}} in filters[0]['or']) ok_({'term': {'app': 2}} in filters[0]['or']) qs = (Addon.search().filter(type=1) .extra(filter={'or_': {'status': 1, 'app': 2}})) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, # {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}]}) eq_(filters.keys(), ['and']) ok_({'term': {'type': 1}} in filters['and']) or_clause = sorted(filters['and'])[0] eq_(or_clause.keys(), ['or']) ok_({'term': {'status': 1}} in or_clause['or']) ok_({'term': {'app': 2}} in or_clause['or'])
def test_extra_filter_or(self): qs = Addon.search().extra(filter={'or_': {'status': 1, 'app': 2}}) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # [{'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}]) assert len(filters) == 1 assert filters[0].keys() == ['or'] assert {'term': {'status': 1}} in filters[0]['or'] assert {'term': {'app': 2}} in filters[0]['or'] qs = (Addon.search().filter(type=1) .extra(filter={'or_': {'status': 1, 'app': 2}})) filters = qs._build_query()['query']['filtered']['filter'] # Filters: # {'and': [{'term': {'type': 1}}, # {'or': [{'term': {'status': 1}}, {'term': {'app': 2}}]}]}) assert filters.keys() == ['and'] assert {'term': {'type': 1}} in filters['and'] or_clause = sorted(filters['and'])[0] assert or_clause.keys() == ['or'] assert {'term': {'status': 1}} in or_clause['or'] assert {'term': {'app': 2}} in or_clause['or']
def test_query_or_and(self): qs = Addon.search().query(or_=dict(type=1, status__gte=2), category=2) query = qs._build_query()['query']['function_score']['query'] # Query: # {'bool': {'must': [{'term': {'category': 2}}, # {'bool': {'should': [ # {'term': {'type': 1}}, # {'range': {'status': {'gte': 2}}}, ]}}]}}) eq_(query.keys(), ['bool']) eq_(query['bool'].keys(), ['must']) ok_({'term': {'category': 2}} in query['bool']['must']) sub_clause = sorted(query['bool']['must'])[0] eq_(sub_clause.keys(), ['bool']) eq_(sub_clause['bool'].keys(), ['should']) ok_({'range': {'status': {'gte': 2}}} in sub_clause['bool']['should']) ok_({'term': {'type': 1}} in sub_clause['bool']['should'])
def test_aggregations(self): Tag(tag_text='sky').save_tag(self._addons[0]) Tag(tag_text='sky').save_tag(self._addons[1]) Tag(tag_text='sky').save_tag(self._addons[2]) Tag(tag_text='earth').save_tag(self._addons[0]) Tag(tag_text='earth').save_tag(self._addons[1]) Tag(tag_text='ocean').save_tag(self._addons[0]) self.reindex(Addon) qs = Addon.search().aggregate(tags={'terms': {'field': 'tags'}}) results = list(qs) assert len(results) == 6 assert qs.aggregations == { u'tags': [ {u'doc_count': 3, u'key': u'sky'}, {u'doc_count': 2, u'key': u'earth'}, {u'doc_count': 1, u'key': u'ocean'}]}
def themes_search(request): search_form = forms.ThemeSearchForm(request.GET) if search_form.is_valid(): q = search_form.cleaned_data['q'] rereview = search_form.cleaned_data['queue_type'] == 'rereview' flagged = search_form.cleaned_data['queue_type'] == 'flagged' # ES query on name. themes = Addon.search().filter(type=amo.ADDON_PERSONA) if rereview: themes = themes.filter(has_theme_rereview=True) else: themes = themes.filter(status=(amo.STATUS_REVIEW_PENDING if flagged else amo.STATUS_PENDING), has_theme_rereview=False) themes = themes.filter_query_string(q)[:100] now = datetime.datetime.now() reviewers = [] for theme in themes: try: themelock = theme.persona.themelock if themelock.expiry > now: reviewers.append(themelock.reviewer.email) else: reviewers.append('') except ObjectDoesNotExist: reviewers.append('') themes = list(themes.values_dict('name', 'slug', 'status')) for theme, reviewer in zip(themes, reviewers): # Collapse single value fields from a list. theme['id'] = theme['id'] theme['slug'] = theme['slug'] theme['status'] = theme['status'] # Dehydrate. theme['reviewer'] = reviewer return {'objects': themes, 'meta': {'total_count': len(themes)}}
def test_getitem(self): addons = list(Addon.search()) eq_(addons[0], Addon.search()[0])
def test_iter(self): qs = Addon.search().filter(type=1, is_disabled=False) eq_(len(qs), len(list(qs)))
def test_count(self): eq_(Addon.search().count(), 6)
def test_count_uses_cached_results(self): qs = Addon.search() qs._results_cache = mock.Mock() qs._results_cache.count = mock.sentinel.count eq_(qs.count(), mock.sentinel.count)
def test_len(self): qs = Addon.search() qs._results_cache = [1] eq_(len(qs), 1)
def test_not_indexed(self): addon = Addon.objects.create(type=amo.ADDON_EXTENSION, status=amo.STATUS_PUBLIC) assert issubclass( Addon.search().filter(id__in=addon.id).count().__class__, mock.Mock)
def test_slice_stop(self): qs = Addon.search()[:6] eq_(qs._build_query()['size'], 6)
def test_order_by_multiple(self): qs = Addon.search().order_by('-rating', 'id') eq_(qs._build_query()['sort'], [{'rating': 'desc'}, 'id'])
def test_values_dict_result(self): addons = [{'id': [a.id], 'slug': [a.slug]} for a in self._addons] qs = Addon.search().values_dict('slug').order_by('id') eq_(list(qs), list(addons))
def test_slice_stop_zero(self): qs = Addon.search()[:0] eq_(qs._build_query()['size'], 0)
def test_empty_values_dict_result(self): qs = Addon.search().values_dict() # Look for some of the keys we expect. for key in ('id', 'name', 'status', 'app'): assert key in qs[0].keys(), qs[0].keys()
def test_object_result(self): qs = Addon.search().filter(id=self._addons[0].id)[:1] eq_(self._addons[:1], list(qs))
def test_object_result_slice(self): addon = self._addons[0] qs = Addon.search().filter(id=addon.id) eq_(addon, qs[0])
def test_extra_bad_key(self): with self.assertRaises(AssertionError): Addon.search().extra(x=1)
def test_extra_values_dict(self): qs = Addon.search().extra(values_dict=['name']) eq_(qs._build_query()['fields'], ['id', 'name']) qs = Addon.search().values_dict('status').extra(values_dict=['name']) eq_(qs._build_query()['fields'], ['id', 'status', 'name'])
def test_empty_values_dict(self): qs = Addon.search().values_dict() assert 'fields' not in qs._build_query()
def test_values_dict(self): qs = Addon.search().values_dict('name') eq_(qs._build_query()['fields'], ['id', 'name'])
def test_slice(self): qs = Addon.search()[5:12] eq_(qs._build_query()['from'], 5) eq_(qs._build_query()['size'], 7)
def test_prefix(self): qs = Addon.search().query(name__startswith='woo') eq_(qs._build_query()['query']['function_score']['query'], {'prefix': { 'name': 'woo' }})
def test_order_by_desc(self): qs = Addon.search().order_by('-rating') eq_(qs._build_query()['sort'], [{'rating': 'desc'}])
def test_order_by_asc(self): qs = Addon.search().order_by('rating') eq_(qs._build_query()['sort'], ['rating'])