def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
device = self._get_device(request)
app_filters = {'profile': get_feature_profile(request)}
if device and device != amo.DEVICE_DESKTOP:
app_filters['device'] = device.id
qs = WebappIndexer.get_app_filter(request, app_filters)
qs = qs.filter('term', **{'collection.id': obj.pk})
qs = qs.sort({
'collection.order': {
'order': 'asc',
'nested_filter': {
'term': {
'collection.id': obj.pk
}
}
}
})
return self.to_native(qs, use_es=True)
def get(self, request, *args, **kwargs):
form_data = self.get_search_data(request, ApiSearchForm)
query = form_data.get('q', '')
base_filters = {'type': form_data['type']}
qs = self.get_query(request, base_filters=base_filters,
region=self.get_region(request))
profile = get_feature_profile(request)
qs = self.apply_filters(request, qs, data=form_data, profile=profile)
names = []
descriptions = []
urls = []
icons = []
for obj in qs:
# FIXME: this does a lot of stuff we don't need. When es_app_to_dict
# is replaced by a Serializer, then we should replace this with a
# custom, lean serializer.
base_data = self.serialize(request, obj)
names.append(base_data['name'])
descriptions.append(truncate(base_data['description']))
urls.append(base_data['absolute_url'])
icons.append(base_data['icons'][64])
return Response([query, names, descriptions, urls, icons],
content_type='application/x-suggestions+json')
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
device = self._get_device(request)
app_filters = {'profile': get_feature_profile(request)}
if device and device != amo.DEVICE_DESKTOP:
app_filters['device'] = device.id
qs = WebappIndexer.get_app_filter(request, app_filters)
qs = qs.filter('term', **{'collection.id': obj.pk})
qs = qs.sort({
'collection.order': {
'order': 'asc',
'nested_filter': {
'term': {'collection.id': obj.pk}
}
}
})
return self.to_native(qs, use_es=True)
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
profile = get_feature_profile(request)
region = self.context['view'].get_region(request)
platform = mkt.PLATFORM_LOOKUP.get(request.GET.get('dev'))
_rget = lambda d: getattr(request, d, False)
qs = Webapp.from_search(request, region=region)
filters = {'collection.id': obj.pk}
filters.update(**self._get_filters(request, es=True))
if profile:
filters.update(**profile.to_kwargs(prefix='features.has_'))
qs = qs.filter(**filters).order_by({
'collection.order': {
'order': 'asc',
'nested_filter': {
'term': {'collection.id': obj.pk}
}
}
})
return self.to_native_es(qs)
def field_to_native(self, obj, field_name):
if not hasattr(self, 'context') or not 'request' in self.context:
raise ImproperlyConfigured('Pass request in self.context when'
' using CollectionMembershipField.')
request = self.context['request']
# Having 'use-es-for-apps' in the context means the parent view wants
# us to use ES to fetch the apps. If that key is present, check that we
# have a view in the context and that the waffle flag is active. If
# everything checks out, bypass the db and use ES to fetch apps for a
# nice performance boost.
if self.context.get('use-es-for-apps') and self.context.get('view'):
return self.field_to_native_es(obj, request)
qs = get_component(obj, self.source)
# Filter apps based on device and feature profiles.
device = self._get_device(request)
profile = get_feature_profile(request)
if device and device != amo.DEVICE_DESKTOP:
qs = qs.filter(addondevicetype__device_type=device.id)
if profile:
qs = qs.filter(**profile.to_kwargs(
prefix='_current_version__features__has_'))
return self.to_native(qs)
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
profile = get_feature_profile(request)
region = self.context['view'].get_region(request)
device = self._get_device(request)
_rget = lambda d: getattr(request, d, False)
qs = Webapp.from_search(request,
region=region,
gaia=_rget('GAIA'),
mobile=_rget('MOBILE'),
tablet=_rget('TABLET'))
filters = {'collection.id': obj.pk}
if device and device != amo.DEVICE_DESKTOP:
filters['device'] = device.id
if profile:
filters.update(**profile.to_kwargs(prefix='features.has_'))
qs = qs.filter(**filters).order_by({
'collection.order': {
'order': 'asc',
'nested_filter': {
'term': {
'collection.id': obj.pk
}
}
}
})
return self.to_native_es(qs)
def field_to_native(self, obj, field_name):
if not hasattr(self, 'context') or not 'request' in self.context:
raise ImproperlyConfigured('Pass request in self.context when'
' using CollectionMembershipField.')
request = self.context['request']
# Having 'use-es-for-apps' in the context means the parent view wants
# us to use ES to fetch the apps. If that key is present, check that we
# have a view in the context and that the waffle flag is active. If
# everything checks out, bypass the db and use ES to fetch apps for a
# nice performance boost.
if self.context.get('use-es-for-apps') and self.context.get('view'):
return self.field_to_native_es(obj, request)
qs = get_component(obj, self.source)
# Filter apps based on device and feature profiles.
device = self._get_device(request)
profile = get_feature_profile(request)
if device and device != amo.DEVICE_DESKTOP:
qs = qs.filter(addondevicetype__device_type=device.id)
if profile:
qs = qs.filter(**profile.to_kwargs(
prefix='_current_version__features__has_'))
return self.to_native(qs)
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
profile = get_feature_profile(request)
region = self.context['view'].get_region_from_request(request)
device = self._get_device(request)
_rget = lambda d: getattr(request, d, False)
qs = Webapp.from_search(request, region=region, gaia=_rget('GAIA'),
mobile=_rget('MOBILE'), tablet=_rget('TABLET'))
filters = {'collection.id': obj.pk}
if device and device != amo.DEVICE_DESKTOP:
filters['device'] = device.id
if profile:
filters.update(**profile.to_kwargs(prefix='features.has_'))
qs = qs.filter(**filters).order_by({
'collection.order': {
'order': 'asc',
'nested_filter': {
'term': {'collection.id': obj.pk}
}
}
})
return self.to_native(qs, use_es=True)
def field_to_native(self, obj, field_name):
if not hasattr(self, 'context') or not 'request' in self.context:
raise ImproperlyConfigured('Pass request in self.context when'
' using CollectionMembershipField.')
request = self.context['request']
# Having 'use-es-for-apps' in the context means the parent view wants us
# to use ES to fetch the apps. If that key is present, check that we
# have a view in the context and that the waffle flag is active. If
# everything checks out, bypass the db and use ES to fetch apps for a
# nice performance boost.
if ('use-es-for-apps' in self.context and 'view' in self.context
and waffle.switch_is_active('collections-use-es-for-apps')):
return self.field_to_native_es(obj, request)
qs = get_component(obj, self.source)
# Filter apps based on feature profiles.
profile = get_feature_profile(request)
if profile:
qs = qs.filter(**profile.to_kwargs(
prefix='_current_version__features__has_'))
return [self.to_native(app) for app in qs]
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
profile = get_feature_profile(request)
region = self.context['view'].get_region(request)
qs = Webapp.from_search(request, region=region)
filters = {'collection.id': obj.pk}
if profile:
filters.update(**profile.to_kwargs(prefix='features.has_'))
qs = qs.filter(**filters).order_by({
'collection.order': {
'order': 'asc',
'nested_filter': {
'term': {'collection.id': obj.pk}
}
}
})
return self.to_native_es(qs)
def filter_queryset(self, request, queryset, view):
profile = get_feature_profile(request)
if profile:
must_not = []
for k in profile.to_kwargs(prefix='features.has_').keys():
must_not.append(F('term', **{k: True}))
if must_not:
return queryset.filter(Bool(must_not=must_not))
return queryset
def search(self, request):
form_data = self.get_search_data(request, ApiSearchForm)
base_filters = {"type": form_data["type"]}
qs = self.get_query(request, base_filters=base_filters, region=self.get_region(request))
profile = get_feature_profile(request)
qs = self.apply_filters(request, qs, data=form_data, profile=profile)
page = self.paginate_queryset(qs)
return self.get_pagination_serializer(page)
def filter_queryset(self, request, queryset, view):
profile = get_feature_profile(request)
if profile:
must_not = []
for k in profile.to_kwargs(prefix='features.has_').keys():
must_not.append(F('term', **{k: True}))
if must_not:
return queryset.filter(Bool(must_not=must_not))
return queryset
def search(self, request):
form_data = self.get_search_data(request)
query = form_data.get('q', '')
qs = self.get_query(request,
region=self.get_region_from_request(request))
profile = get_feature_profile(request)
qs = self.apply_filters(request, qs, data=form_data,
profile=profile)
page = self.paginate_queryset(qs)
return self.get_pagination_serializer(page), query
def search(self, request):
form_data = self.get_search_data(request)
query = form_data.get('q', '')
base_filters = {'type': form_data['type']}
qs = self.get_query(request, base_filters=base_filters,
region=self.get_region_from_request(request))
profile = get_feature_profile(request)
qs = self.apply_filters(request, qs, data=form_data,
profile=profile)
page = self.paginate_queryset(qs.values_dict())
return self.get_pagination_serializer(page), query
def search(self, request):
form_data = self.get_search_data(request)
query = form_data.get('q', '')
base_filters = {'type': form_data['type']}
qs = self.get_query(request,
base_filters=base_filters,
region=self.get_region(request))
profile = get_feature_profile(request)
qs = self.apply_filters(request, qs, data=form_data, profile=profile)
page = self.paginate_queryset(qs.values_dict())
return self.get_pagination_serializer(page), query
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a FeaturedSearchView instance in self.context['view']
to properly rehydrate results returned by ES.
"""
profile = get_feature_profile(request)
region = self.context["view"].get_region(request)
qs = Webapp.from_search(request, region=region)
filters = {"collection.id": obj.pk}
if profile:
filters.update(**profile.to_kwargs(prefix="features.has_"))
qs = qs.filter(**filters).order_by("collection.order")
return [self.to_native_es(app) for app in qs]
def field_to_native(self, obj, field_name):
if not hasattr(self, "context") or not "request" in self.context:
raise ImproperlyConfigured("Pass request in self.context when" " using CollectionMembershipField.")
request = self.context["request"]
# If we have a search resource in the context, we should try to use ES
# to fetch the apps, if the waffle flag is active.
if "search_resource" in self.context and waffle.switch_is_active("collections-use-es-for-apps"):
return self.field_to_native_es(obj, request)
value = get_component(obj, self.source)
# Filter apps based on feature profiles.
profile = get_feature_profile(request)
if profile and waffle.switch_is_active("buchets"):
value = value.filter(**profile.to_kwargs(prefix="app___current_version__features__has_"))
return [self.to_native(item) for item in value.all()]
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a SearchResource instance in self.context['search_resource']
to properly rehydrate results returned by ES.
"""
search_resource = self.context["search_resource"]
profile = get_feature_profile(request)
region = search_resource.get_region(request)
qs = Webapp.from_search(request, region=region)
filters = {"collection.id": obj.pk}
if profile and waffle.switch_is_active("buchets"):
filters.update(**profile.to_kwargs(prefix="features.has_"))
qs = qs.filter(**filters).order_by("collection.order")
return [bundle.data for bundle in search_resource.rehydrate_results(request, qs)]
def field_to_native_es(self, obj, request):
"""
A version of field_to_native that uses ElasticSearch to fetch the apps
belonging to the collection instead of SQL.
Relies on a SearchResource instance in self.context['search_resource']
to properly rehydrate results returned by ES.
"""
search_resource = self.context['search_resource']
profile = get_feature_profile(request)
region = search_resource.get_region(request)
qs = Webapp.from_search(request, region=region)
filters = {'collection.id': obj.pk}
if profile:
filters.update(**profile.to_kwargs(prefix='features.has_'))
qs = qs.filter(**filters).order_by('collection.order')
return [bundle.data
for bundle in search_resource.rehydrate_results(request, qs)]
def field_to_native(self, obj, field_name):
if not hasattr(self, 'context') or not 'request' in self.context:
raise ImproperlyConfigured('Pass request in self.context when'
' using CollectionMembershipField.')
request = self.context['request']
# If we have a search resource in the context, we should try to use ES
# to fetch the apps, if the waffle flag is active.
if ('search_resource' in self.context
and waffle.switch_is_active('collections-use-es-for-apps')):
return self.field_to_native_es(obj, request)
value = get_component(obj, self.source)
# Filter apps based on feature profiles.
profile = get_feature_profile(request)
if profile:
value = value.filter(**profile.to_kwargs(
prefix='app___current_version__features__has_'))
return [self.to_native(item) for item in value.all()]
def get(self, request, *args, **kwargs):
form_data = self.get_search_data(request, ApiSearchForm)
query = form_data.get('q', '')
base_filters = {'type': form_data['type']}
qs = self.get_query(request, base_filters=base_filters,
region=self.get_region(request))
profile = get_feature_profile(request)
qs = self.apply_filters(request, qs, data=form_data, profile=profile)
names = []
descriptions = []
urls = []
icons = []
for obj in qs:
base_data = self.serialize(request, obj)
names.append(base_data['name'])
descriptions.append(truncate(base_data['description']))
urls.append(base_data['absolute_url'])
icons.append(base_data['icons'][64])
return Response([query, names, descriptions, urls, icons],
content_type='application/x-suggestions+json')
def get_app_filter(cls, request, additional_data=None, sq=None,
app_ids=None, no_filter=False):
"""
THE grand, consolidated ES filter for Webapps. By default:
- Excludes non-public apps.
- Excludes disabled apps (whether by reviewer or by developer).
- Excludes based on region exclusions.
- TODO: Excludes based on device and platform support.
additional_data -- an object with more data to allow more filtering.
sq -- if you have an existing search object to filter off of.
app_ids -- if you want to filter by a list of app IDs.
no_filter -- doesn't apply the consumer-side excludes (public/region).
"""
from mkt.api.base import get_region_from_request
from mkt.search.views import name_query
sq = sq or cls.search()
additional_data = additional_data or {}
app_ids = app_ids or []
data = {
'app_type': [],
'author.raw': None,
'category': None, # Slug.
'device': None, # ID.
'gaia': getattr(request, 'GAIA', False),
'is_offline': None,
'manifest_url': '',
'mobile': getattr(request, 'MOBILE', False),
'premium_type': [],
'profile': get_feature_profile(request),
'q': '',
'region': getattr(get_region_from_request(request), 'id', None),
'status': None,
'supported_locales': [],
'tablet': getattr(request, 'TABLET', False),
'tags': '',
}
data.update(additional_data)
# Fields that will be filtered with a term query.
term_fields = ('author.raw', 'device', 'manifest_url', 'status',
'tags')
# Fields that will be filtered with a terms query.
terms_fields = ('category', 'premium_type', 'app_type',
'supported_locales')
# QUERY.
if data['q']:
# Function score for popularity boosting (defaults to multiply).
sq = sq.query(
'function_score',
query=name_query(data['q'].lower()),
functions=[query.SF('field_value_factor', field='boost')])
# MUST.
must = [
F('term', status=amo.STATUS_PUBLIC),
F('term', is_disabled=False),
] if not no_filter else []
for field in term_fields + terms_fields:
# Term filters.
if data[field]:
filter_type = 'term' if field in term_fields else 'terms'
must.append(F(filter_type, **{field: data[field]}))
if not no_filter:
if data['profile']:
# Feature filters.
profile = data['profile']
for k, v in profile.to_kwargs(prefix='features.has_').items():
must.append(F('term', **{k: v}))
if data['mobile'] or data['gaia']:
# Uses flash.
must.append(F('term', uses_flash=False))
if data['is_offline'] is not None:
must.append(F('term', is_offline=data['is_offline']))
# SHOULD.
should = []
if app_ids:
should = [es_filter.Terms(id=list(set(app_ids)))]
sq = sq[0:len(set(app_ids))]
# FILTER.
if must or should:
sq = sq.filter(es_filter.Bool(must=must, should=should))
if data['region'] and not no_filter:
# Region exclusions.
sq = sq.filter(~F('term', region_exclusions=data['region']))
return sq
def get_feature_profile(self, request):
# Overridden by reviewers search api to disable profile filtering.
return get_feature_profile(request)
def get_feature_profile(self, request):
# Overridden by reviewers search api to disable profile filtering.
return get_feature_profile(request)
