def test_get_indexes(self): """Test get_indexes always returns a list of strings.""" # Pulls it from ES_INDEXES (list of strings). s = S(FakeDjangoMappingType) eq_(s.get_indexes(), ['elasticutilstest']) # Pulls it from ES_INDEXES (string). old_indexes = settings.ES_INDEXES try: settings.ES_INDEXES = {'default': 'elasticutilstest'} s = S(FakeDjangoMappingType) eq_(s.get_indexes(), ['elasticutilstest']) finally: settings.ES_INDEXES = old_indexes # Pulls from indexes. s = S(FakeDjangoMappingType).indexes('footest') eq_(s.get_indexes(), ['footest']) s = S(FakeDjangoMappingType).indexes('footest', 'footest2') eq_(s.get_indexes(), ['footest', 'footest2']) s = S(FakeDjangoMappingType).indexes('footest').indexes('footest2') eq_(s.get_indexes(), ['footest2'])
def search(cls, query, include_non_vouched=False): """Sensible default search for UserProfiles.""" query = query.lower().strip() fields = ('username', 'bio__text', 'email', 'ircname', 'country__text', 'country__text_phrase', 'region__text', 'region__text_phrase', 'city__text', 'city__text_phrase', 'fullname__text', 'fullname__text_phrase', 'fullname__prefix', 'fullname__fuzzy' 'groups__text') if query: q = dict((field, query) for field in fields) s = (S(cls).boost(fullname__text_phrase=5, username=5, email=5, ircname=5, fullname__text=4, country__text_phrase=4, region__text_phrase=4, city__text_phrase=4, fullname__prefix=3, fullname__fuzzy=2, bio__text=2).query(or_=q)) else: s = S(cls) s = s.order_by('_score', 'name') if not include_non_vouched: s = s.filter(is_vouched=True) return s
def test_filter_or_3(self): eq_(len(S(FakeDjangoMappingType).filter(F(tag='awesome') | F(tag='boat') | F(tag='boring'))), 5) eq_(len(S(FakeDjangoMappingType).filter(or_={'foo': 'bar', 'or_': {'tag': 'boat', 'width': '5'} })), 3)
def test_filter_empty_f(self): eq_(len(S(FakeDjangoMappingType).filter(F() | F(tag='awesome'))), 3) eq_(len(S(FakeDjangoMappingType).filter(F() & F(tag='awesome'))), 3) eq_(len(S(FakeDjangoMappingType).filter(F() | F() | F(tag='awesome'))), 3) eq_(len(S(FakeDjangoMappingType).filter(F() & F() & F(tag='awesome'))), 3) eq_(len(S(FakeDjangoMappingType).filter(F())), 5)
def test_filter_and(self): eq_(len(S(FakeDjangoMappingType).filter(tag='awesome', foo='bar')), 1) eq_( len( S(FakeDjangoMappingType).filter(tag='awesome').filter( foo='bar')), 1) eq_( len( S(FakeDjangoMappingType).filter( F(tag='awesome') & F(foo='bar'))), 1)
def test_facet_raw(self): qs = S(FakeDjangoMappingType).facet_raw(tags={'terms': {'field': 'tag'}}) eq_(facet_counts_dict(qs, 'tags'), dict(awesome=3, boring=1, boat=1)) qs = (S(FakeDjangoMappingType) .query(foo='car') .facet_raw(tags={'terms': {'field': 'tag'}})) eq_(facet_counts_dict(qs, 'tags'), {'awesome': 2})
def test_filter_not(self): eq_(len(S(FakeDjangoMappingType).filter(~F(tag='awesome'))), 2) eq_( len( S(FakeDjangoMappingType).filter(~(F(tag='boring') | F(tag='boat')))), 3) eq_( len( S(FakeDjangoMappingType).filter(~F(tag='boat')).filter(~F( foo='bar'))), 3) eq_(len(S(FakeDjangoMappingType).filter(~F(tag='boat', foo='barf'))), 5)
def test_get_doctypes(self): """Test get_doctypes always returns a list of strings.""" # Pulls from ._meta.db_table. s = S(FakeDjangoMappingType) eq_(s.get_doctypes(), ['fake']) # Pulls from doctypes. s = S(FakeDjangoMappingType).doctypes('footype') eq_(s.get_doctypes(), ['footype']) s = S(FakeDjangoMappingType).doctypes('footype', 'footype2') eq_(s.get_doctypes(), ['footype', 'footype2']) s = S(FakeDjangoMappingType).doctypes('footype').doctypes('footype2') eq_(s.get_doctypes(), ['footype2'])
def app_search(request): results = [] q = request.GET.get('q', u'').lower().strip() fields = ('name', 'app_slug') non_es_fields = ['id', 'name__localized_string'] + list(fields) if q.isnumeric(): qs = (Webapp.objects.filter(pk=q) .values(*non_es_fields)) else: # Try to load by GUID: qs = (Webapp.objects.filter(guid=q) .values(*non_es_fields)) if not qs.count(): qs = (S(WebappIndexer) .query(should=True, **_expand_query(q, fields)) .values_dict(*['id'] + list(fields))) qs = _slice_results(request, qs) for app in qs: if 'name__localized_string' in app: # This is a result from the database. app['url'] = reverse('lookup.app_summary', args=[app['id']]) app['name'] = app['name__localized_string'] results.append(app) else: # This is a result from elasticsearch which returns name as a list. app['url'] = reverse('lookup.app_summary', args=[app['id']]) for field in ('id', 'app_slug'): app[field] = app.get(field) for name in app['name']: dd = app.copy() dd['name'] = name results.append(dd) return {'results': results}
def get_doctype_stats(index): """Returns a dict of name -> count for documents indexed. For example: >>> get_doctype_stats() {'questions_question': 14216, 'forums_thread': 419, 'wiki_document': 759} :throws elasticsearch.exceptions.ConnectionError: if there is a connection error, including a timeout. :throws elasticsearch.exceptions.NotFound: if the index doesn't exist """ stats = {} from kitsune.search.models import get_mapping_types for cls in get_mapping_types(): if cls.get_index() == index: # Note: Can't use cls.search() here since that returns a # Sphilastic which is hard-coded to look only at the # read index.. s = S(cls).indexes(index) stats[cls.get_mapping_type_name()] = s.count() return stats
def get_index_stats(): """Return dict of name -> count for documents indexed. For example: >>> get_index_stats() {'response': 122233} .. Note:: This infers the index to use from the registered mapping types. :returns: mapping type name -> count for documents indexes. :throws pyelasticsearch.exceptions.Timeout: if the request times out :throws pyelasticsearch.exceptions.ConnectionError: if there's a connection error :throws pyelasticsearch.exceptions.ElasticHttpNotFound: if the index doesn't exist """ stats = {} for name, cls in get_mapping_types().items(): stats[name] = S(cls).count() return stats
def search(cls): """Returns a typed S for this class. :returns: an `S` """ return S(cls)
def app_search(request): results = [] q = request.GET.get('q', u'').lower().strip() addon_type = request.GET.get('type', amo.ADDON_WEBAPP) fields = ('name', 'app_slug') non_es_fields = ['id', 'name__localized_string'] + list(fields) if q.isnumeric(): qs = (Addon.objects.filter(type=addon_type, pk=q).values(*non_es_fields)) else: # Try to load by GUID: qs = (Addon.objects.filter(type=addon_type, guid=q).values(*non_es_fields)) if not qs.count(): qs = (S(Addon).query(type=addon_type, or_=_expand_query( q, fields)).values_dict(*fields)[:20]) for app in qs: app['url'] = reverse('lookup.app_summary', args=[app['id']]) # ES returns a list of localized names but database queries do not. if type(app['name']) != list: app['name'] = [app['name__localized_string']] for name in app['name']: dd = app.copy() dd['name'] = name results.append(dd) return {'results': results}
def test_facet_raw_overrides_facet(self): """facet_raw overrides facet with the same facet name.""" qs = (S(FakeDjangoMappingType) .query(foo='car') .facet('tag') .facet_raw(tag={'terms': {'field': 'tag'}, 'global': True})) eq_(facet_counts_dict(qs, 'tag'), dict(awesome=3, boring=1, boat=1))
def test_global_facet(self): qs = S(FakeDjangoMappingType).query(foo='car').filter(width=5) # facet restricted to query eq_(facet_counts_dict(qs.facet('tag'), 'tag'), {'awesome': 2}) # facet applies to all of corpus eq_(facet_counts_dict(qs.facet('tag', global_=True), 'tag'), dict(awesome=3, boring=1, boat=1))
def test_filtered_facet(self): qs = S(FakeDjangoMappingType).query(foo='car').filter(width=5) # filter doesn't apply to facets eq_(facet_counts_dict(qs.facet('tag'), 'tag'), {'awesome': 2}) # filter does apply to facets eq_(facet_counts_dict(qs.facet('tag', filtered=True), 'tag'), {'awesome': 1})
def search(cls, query, vouched=None, photo=None): """Sensible default search for UserProfiles.""" query = query.lower().strip() fields = ('username', 'bio__text', 'website', 'email', 'groups', 'skills', 'languages', 'first_name__prefix', 'last_name__prefix', 'ircname', 'country', 'region', 'city') if query: q = dict((field, query) for field in fields) s = S(cls).query(or_=q) else: s = S(cls) if vouched is not None: s = s.filter(is_vouched=vouched) if photo is not None: s = s.filter(has_photo=photo) return s
def test_index(self): document = {'id': 1, 'name': 'odin skullcrusher'} # Generate the FakeModel in our "database" FakeModel(**document) # Index the document with .index() FakeDjangoMappingType.index(document, id_=document['id']) IndexableTest.refresh() # Query it to make sure it's there. eq_(len(S(FakeDjangoMappingType).query(name__prefix='odin')), 1)
def test_index(self): self.persist_data([ { 'id': 1, 'name': 'odin skullcrusher' }, { 'id': 2, 'name': 'olaf bloodbiter' }, ]) # Query it to make sure it's there. eq_(len(S(FakeDjangoMappingType).query(name__prefix='odin')), 1)
def test_bulk_index(self): documents = [{ 'id': 1, 'name': 'odin skullcrusher' }, { 'id': 2, 'name': 'heimdall kneebiter' }, { 'id': 3, 'name': 'erik rose' }] # Generate the FakeModel in our "database" for doc in documents: FakeModel(**doc) # Index the document with .index() FakeDjangoMappingType.bulk_index(documents, id_field='id') IndexableTest.refresh() # Query it to make sure they're there. eq_(len(S(FakeDjangoMappingType).query(name__prefix='odin')), 1) eq_(len(S(FakeDjangoMappingType).query(name__prefix='erik')), 1)
def test_get_object(self): self.persist_data([ { 'id': 1, 'name': 'odin skullcrusher' }, { 'id': 2, 'name': 'olaf bloodbiter' }, ]) s = S(FakeDjangoMappingType).query(name__prefix='odin') obj = s[0] eq_(obj.object.id, 1)
def featured_suggestions(request): q = request.GET.get('q', u'').lower().strip() cat_slug = request.GET.get('category') filters = { 'type': amo.ADDON_WEBAPP, 'status': amo.STATUS_PUBLIC, 'is_disabled': False, } if cat_slug: filters.update({'category': cat_slug}) search_fields = ['app_slug', 'name'] # If search looks like an ID, also search the ID field. if q.isdigit(): qs = search_fields.append('id') # Do a search based on the query string. qs = S(WebappIndexer).filter(**filters).query(should=True, **dict( ('{0}__prefix'.format(f), w) for f in search_fields for w in q.split())) fields = ['id', 'app_slug', 'default_locale'] for analyzer in amo.SEARCH_ANALYZER_MAP: if (not settings.ES_USE_PLUGINS and analyzer in amo.SEARCH_ANALYZER_PLUGINS): continue fields.append('name_{0}'.format(analyzer)) qs = qs.values_dict(*fields) results = [] for app in qs[:20]: results.append({ 'id': app._id, 'name': get_attr_lang(app, 'name', app['default_locale']), 'url': reverse('detail', args=[app['app_slug']]), }) return results
def from_search(cls, cat=None, region=None, gaia=False): filters = dict(type=amo.ADDON_WEBAPP, status=amo.STATUS_PUBLIC, is_disabled=False) if cat: filters.update(category=cat.id) srch = S(cls).query(**filters) if region: excluded = cls.get_excluded_in(region) if excluded: srch = srch.filter(~F(id__in=excluded)) if waffle.switch_is_active('disabled-payments') or not gaia: srch = srch.filter(premium_type__in=amo.ADDON_FREES, price=0) return srch
def from_search(cls, cat=None, region=None, gaia=False, mobile=False, tablet=False, filter_overrides=None): filters = dict(type=amo.ADDON_WEBAPP, status=amo.STATUS_PUBLIC, is_disabled=False) # Special handling if status is 'any' to remove status filter. if filter_overrides and 'status' in filter_overrides: if filter_overrides['status'] is 'any': del filters['status'] del filter_overrides['status'] if filter_overrides: filters.update(filter_overrides) if cat: filters.update(category=cat.id) srch = S(cls).query(**filters) if region: excluded = cls.get_excluded_in(region) if excluded: srch = srch.filter(~F(id__in=excluded)) if mobile: srch = srch.filter(uses_flash=False) if (mobile or tablet) and not gaia: # Don't show packaged apps on Firefox for Android. srch = srch.filter(app_type=amo.ADDON_WEBAPP_HOSTED) # Only show premium apps on gaia and desktop for now. srch = srch.filter( ~F(premium_type__in=amo.ADDON_PREMIUMS, price__gt=0)) return srch
def apply_filters(self, request, applicable_filters): """Implement advanced filters. - Implement 'groups' filter. - Implement 'languages' filter. - Implement 'skills' filter. """ if (request.GET.get('restricted', False) and 'email__text' not in applicable_filters and len(applicable_filters) != 1): raise ImmediateHttpResponse(response=http.HttpForbidden()) if request.GET.get('restricted', False): applicable_filters.append(F(allows_community_sites=True)) mega_filter = F() for filter in applicable_filters: mega_filter &= filter return S(UserProfile).filter(mega_filter)
def queryset(self): """Get items based on ID or search by name.""" results = Addon.objects.none() q = self.request.GET.get(self.key) if q: pk = None try: pk = int(q) except ValueError: pass qs = None if pk: qs = Addon.objects.filter(id=int(q), disabled_by_user=False) elif len(q) > 2: # Oh, how I wish I could elastically exclude terms. # (You can now, but I forgot why I was complaining to # begin with.) qs = (S(Addon).query(or_=name_only_query(q.lower())).filter( is_disabled=False)) if qs: results = qs.filter(type__in=self.types, status__in=amo.REVIEWED_STATUSES) return results
def test_filter_bad_field_action(self): with self.assertRaises(InvalidFieldActionError): len(S(FakeDjangoMappingType).filter(F(tag__faux='awesome')))
def test_order_by(self): res = S(FakeDjangoMappingType).filter(tag='awesome').order_by('-width') eq_([d.id for d in res], [5, 3, 1])
def get_obj(self): return S(WebappIndexer).filter(id=self.app.pk).execute().objects[0]
def test_facet(self): qs = S(FakeDjangoMappingType).facet('tag') eq_(facet_counts_dict(qs, 'tag'), dict(awesome=3, boring=1, boat=1))