def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ device = self._get_device(request) app_filters = {'profile': get_feature_profile(request)} if device and device != amo.DEVICE_DESKTOP: app_filters['device'] = device.id qs = WebappIndexer.get_app_filter(request, app_filters) qs = qs.filter('term', **{'collection.id': obj.pk}) qs = qs.sort({ 'collection.order': { 'order': 'asc', 'nested_filter': { 'term': { 'collection.id': obj.pk } } } }) return self.to_native(qs, use_es=True)
def get(self, request, *args, **kwargs): form_data = self.get_search_data(request, ApiSearchForm) query = form_data.get('q', '') base_filters = {'type': form_data['type']} qs = self.get_query(request, base_filters=base_filters, region=self.get_region(request)) profile = get_feature_profile(request) qs = self.apply_filters(request, qs, data=form_data, profile=profile) names = [] descriptions = [] urls = [] icons = [] for obj in qs: # FIXME: this does a lot of stuff we don't need. When es_app_to_dict # is replaced by a Serializer, then we should replace this with a # custom, lean serializer. base_data = self.serialize(request, obj) names.append(base_data['name']) descriptions.append(truncate(base_data['description'])) urls.append(base_data['absolute_url']) icons.append(base_data['icons'][64]) return Response([query, names, descriptions, urls, icons], content_type='application/x-suggestions+json')
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ device = self._get_device(request) app_filters = {'profile': get_feature_profile(request)} if device and device != amo.DEVICE_DESKTOP: app_filters['device'] = device.id qs = WebappIndexer.get_app_filter(request, app_filters) qs = qs.filter('term', **{'collection.id': obj.pk}) qs = qs.sort({ 'collection.order': { 'order': 'asc', 'nested_filter': { 'term': {'collection.id': obj.pk} } } }) return self.to_native(qs, use_es=True)
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ profile = get_feature_profile(request) region = self.context['view'].get_region(request) platform = mkt.PLATFORM_LOOKUP.get(request.GET.get('dev')) _rget = lambda d: getattr(request, d, False) qs = Webapp.from_search(request, region=region) filters = {'collection.id': obj.pk} filters.update(**self._get_filters(request, es=True)) if profile: filters.update(**profile.to_kwargs(prefix='features.has_')) qs = qs.filter(**filters).order_by({ 'collection.order': { 'order': 'asc', 'nested_filter': { 'term': {'collection.id': obj.pk} } } }) return self.to_native_es(qs)
def field_to_native(self, obj, field_name): if not hasattr(self, 'context') or not 'request' in self.context: raise ImproperlyConfigured('Pass request in self.context when' ' using CollectionMembershipField.') request = self.context['request'] # Having 'use-es-for-apps' in the context means the parent view wants # us to use ES to fetch the apps. If that key is present, check that we # have a view in the context and that the waffle flag is active. If # everything checks out, bypass the db and use ES to fetch apps for a # nice performance boost. if self.context.get('use-es-for-apps') and self.context.get('view'): return self.field_to_native_es(obj, request) qs = get_component(obj, self.source) # Filter apps based on device and feature profiles. device = self._get_device(request) profile = get_feature_profile(request) if device and device != amo.DEVICE_DESKTOP: qs = qs.filter(addondevicetype__device_type=device.id) if profile: qs = qs.filter(**profile.to_kwargs( prefix='_current_version__features__has_')) return self.to_native(qs)
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ profile = get_feature_profile(request) region = self.context['view'].get_region(request) device = self._get_device(request) _rget = lambda d: getattr(request, d, False) qs = Webapp.from_search(request, region=region, gaia=_rget('GAIA'), mobile=_rget('MOBILE'), tablet=_rget('TABLET')) filters = {'collection.id': obj.pk} if device and device != amo.DEVICE_DESKTOP: filters['device'] = device.id if profile: filters.update(**profile.to_kwargs(prefix='features.has_')) qs = qs.filter(**filters).order_by({ 'collection.order': { 'order': 'asc', 'nested_filter': { 'term': { 'collection.id': obj.pk } } } }) return self.to_native_es(qs)
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ profile = get_feature_profile(request) region = self.context['view'].get_region_from_request(request) device = self._get_device(request) _rget = lambda d: getattr(request, d, False) qs = Webapp.from_search(request, region=region, gaia=_rget('GAIA'), mobile=_rget('MOBILE'), tablet=_rget('TABLET')) filters = {'collection.id': obj.pk} if device and device != amo.DEVICE_DESKTOP: filters['device'] = device.id if profile: filters.update(**profile.to_kwargs(prefix='features.has_')) qs = qs.filter(**filters).order_by({ 'collection.order': { 'order': 'asc', 'nested_filter': { 'term': {'collection.id': obj.pk} } } }) return self.to_native(qs, use_es=True)
def field_to_native(self, obj, field_name): if not hasattr(self, 'context') or not 'request' in self.context: raise ImproperlyConfigured('Pass request in self.context when' ' using CollectionMembershipField.') request = self.context['request'] # Having 'use-es-for-apps' in the context means the parent view wants us # to use ES to fetch the apps. If that key is present, check that we # have a view in the context and that the waffle flag is active. If # everything checks out, bypass the db and use ES to fetch apps for a # nice performance boost. if ('use-es-for-apps' in self.context and 'view' in self.context and waffle.switch_is_active('collections-use-es-for-apps')): return self.field_to_native_es(obj, request) qs = get_component(obj, self.source) # Filter apps based on feature profiles. profile = get_feature_profile(request) if profile: qs = qs.filter(**profile.to_kwargs( prefix='_current_version__features__has_')) return [self.to_native(app) for app in qs]
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ profile = get_feature_profile(request) region = self.context['view'].get_region(request) qs = Webapp.from_search(request, region=region) filters = {'collection.id': obj.pk} if profile: filters.update(**profile.to_kwargs(prefix='features.has_')) qs = qs.filter(**filters).order_by({ 'collection.order': { 'order': 'asc', 'nested_filter': { 'term': {'collection.id': obj.pk} } } }) return self.to_native_es(qs)
def filter_queryset(self, request, queryset, view): profile = get_feature_profile(request) if profile: must_not = [] for k in profile.to_kwargs(prefix='features.has_').keys(): must_not.append(F('term', **{k: True})) if must_not: return queryset.filter(Bool(must_not=must_not)) return queryset
def search(self, request): form_data = self.get_search_data(request, ApiSearchForm) base_filters = {"type": form_data["type"]} qs = self.get_query(request, base_filters=base_filters, region=self.get_region(request)) profile = get_feature_profile(request) qs = self.apply_filters(request, qs, data=form_data, profile=profile) page = self.paginate_queryset(qs) return self.get_pagination_serializer(page)
def search(self, request): form_data = self.get_search_data(request) query = form_data.get('q', '') qs = self.get_query(request, region=self.get_region_from_request(request)) profile = get_feature_profile(request) qs = self.apply_filters(request, qs, data=form_data, profile=profile) page = self.paginate_queryset(qs) return self.get_pagination_serializer(page), query
def search(self, request): form_data = self.get_search_data(request) query = form_data.get('q', '') base_filters = {'type': form_data['type']} qs = self.get_query(request, base_filters=base_filters, region=self.get_region_from_request(request)) profile = get_feature_profile(request) qs = self.apply_filters(request, qs, data=form_data, profile=profile) page = self.paginate_queryset(qs.values_dict()) return self.get_pagination_serializer(page), query
def search(self, request): form_data = self.get_search_data(request) query = form_data.get('q', '') base_filters = {'type': form_data['type']} qs = self.get_query(request, base_filters=base_filters, region=self.get_region(request)) profile = get_feature_profile(request) qs = self.apply_filters(request, qs, data=form_data, profile=profile) page = self.paginate_queryset(qs.values_dict()) return self.get_pagination_serializer(page), query
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a FeaturedSearchView instance in self.context['view'] to properly rehydrate results returned by ES. """ profile = get_feature_profile(request) region = self.context["view"].get_region(request) qs = Webapp.from_search(request, region=region) filters = {"collection.id": obj.pk} if profile: filters.update(**profile.to_kwargs(prefix="features.has_")) qs = qs.filter(**filters).order_by("collection.order") return [self.to_native_es(app) for app in qs]
def field_to_native(self, obj, field_name): if not hasattr(self, "context") or not "request" in self.context: raise ImproperlyConfigured("Pass request in self.context when" " using CollectionMembershipField.") request = self.context["request"] # If we have a search resource in the context, we should try to use ES # to fetch the apps, if the waffle flag is active. if "search_resource" in self.context and waffle.switch_is_active("collections-use-es-for-apps"): return self.field_to_native_es(obj, request) value = get_component(obj, self.source) # Filter apps based on feature profiles. profile = get_feature_profile(request) if profile and waffle.switch_is_active("buchets"): value = value.filter(**profile.to_kwargs(prefix="app___current_version__features__has_")) return [self.to_native(item) for item in value.all()]
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a SearchResource instance in self.context['search_resource'] to properly rehydrate results returned by ES. """ search_resource = self.context["search_resource"] profile = get_feature_profile(request) region = search_resource.get_region(request) qs = Webapp.from_search(request, region=region) filters = {"collection.id": obj.pk} if profile and waffle.switch_is_active("buchets"): filters.update(**profile.to_kwargs(prefix="features.has_")) qs = qs.filter(**filters).order_by("collection.order") return [bundle.data for bundle in search_resource.rehydrate_results(request, qs)]
def field_to_native_es(self, obj, request): """ A version of field_to_native that uses ElasticSearch to fetch the apps belonging to the collection instead of SQL. Relies on a SearchResource instance in self.context['search_resource'] to properly rehydrate results returned by ES. """ search_resource = self.context['search_resource'] profile = get_feature_profile(request) region = search_resource.get_region(request) qs = Webapp.from_search(request, region=region) filters = {'collection.id': obj.pk} if profile: filters.update(**profile.to_kwargs(prefix='features.has_')) qs = qs.filter(**filters).order_by('collection.order') return [bundle.data for bundle in search_resource.rehydrate_results(request, qs)]
def field_to_native(self, obj, field_name): if not hasattr(self, 'context') or not 'request' in self.context: raise ImproperlyConfigured('Pass request in self.context when' ' using CollectionMembershipField.') request = self.context['request'] # If we have a search resource in the context, we should try to use ES # to fetch the apps, if the waffle flag is active. if ('search_resource' in self.context and waffle.switch_is_active('collections-use-es-for-apps')): return self.field_to_native_es(obj, request) value = get_component(obj, self.source) # Filter apps based on feature profiles. profile = get_feature_profile(request) if profile: value = value.filter(**profile.to_kwargs( prefix='app___current_version__features__has_')) return [self.to_native(item) for item in value.all()]
def get(self, request, *args, **kwargs): form_data = self.get_search_data(request, ApiSearchForm) query = form_data.get('q', '') base_filters = {'type': form_data['type']} qs = self.get_query(request, base_filters=base_filters, region=self.get_region(request)) profile = get_feature_profile(request) qs = self.apply_filters(request, qs, data=form_data, profile=profile) names = [] descriptions = [] urls = [] icons = [] for obj in qs: base_data = self.serialize(request, obj) names.append(base_data['name']) descriptions.append(truncate(base_data['description'])) urls.append(base_data['absolute_url']) icons.append(base_data['icons'][64]) return Response([query, names, descriptions, urls, icons], content_type='application/x-suggestions+json')
def get_app_filter(cls, request, additional_data=None, sq=None, app_ids=None, no_filter=False): """ THE grand, consolidated ES filter for Webapps. By default: - Excludes non-public apps. - Excludes disabled apps (whether by reviewer or by developer). - Excludes based on region exclusions. - TODO: Excludes based on device and platform support. additional_data -- an object with more data to allow more filtering. sq -- if you have an existing search object to filter off of. app_ids -- if you want to filter by a list of app IDs. no_filter -- doesn't apply the consumer-side excludes (public/region). """ from mkt.api.base import get_region_from_request from mkt.search.views import name_query sq = sq or cls.search() additional_data = additional_data or {} app_ids = app_ids or [] data = { 'app_type': [], 'author.raw': None, 'category': None, # Slug. 'device': None, # ID. 'gaia': getattr(request, 'GAIA', False), 'is_offline': None, 'manifest_url': '', 'mobile': getattr(request, 'MOBILE', False), 'premium_type': [], 'profile': get_feature_profile(request), 'q': '', 'region': getattr(get_region_from_request(request), 'id', None), 'status': None, 'supported_locales': [], 'tablet': getattr(request, 'TABLET', False), 'tags': '', } data.update(additional_data) # Fields that will be filtered with a term query. term_fields = ('author.raw', 'device', 'manifest_url', 'status', 'tags') # Fields that will be filtered with a terms query. terms_fields = ('category', 'premium_type', 'app_type', 'supported_locales') # QUERY. if data['q']: # Function score for popularity boosting (defaults to multiply). sq = sq.query( 'function_score', query=name_query(data['q'].lower()), functions=[query.SF('field_value_factor', field='boost')]) # MUST. must = [ F('term', status=amo.STATUS_PUBLIC), F('term', is_disabled=False), ] if not no_filter else [] for field in term_fields + terms_fields: # Term filters. if data[field]: filter_type = 'term' if field in term_fields else 'terms' must.append(F(filter_type, **{field: data[field]})) if not no_filter: if data['profile']: # Feature filters. profile = data['profile'] for k, v in profile.to_kwargs(prefix='features.has_').items(): must.append(F('term', **{k: v})) if data['mobile'] or data['gaia']: # Uses flash. must.append(F('term', uses_flash=False)) if data['is_offline'] is not None: must.append(F('term', is_offline=data['is_offline'])) # SHOULD. should = [] if app_ids: should = [es_filter.Terms(id=list(set(app_ids)))] sq = sq[0:len(set(app_ids))] # FILTER. if must or should: sq = sq.filter(es_filter.Bool(must=must, should=should)) if data['region'] and not no_filter: # Region exclusions. sq = sq.filter(~F('term', region_exclusions=data['region'])) return sq
def get_feature_profile(self, request): # Overridden by reviewers search api to disable profile filtering. return get_feature_profile(request)