def filter_queryset(self, request, queryset, view):
region = get_region_from_request(request)
search_query = request.GET.get("q")
sort = request.GET.getlist("sort")
# When querying (with `?q=`) we want to sort by relevance. If no query
# is provided and no `?sort` is provided, i.e. we are only applying
# filters which don't affect the relevance, we sort by popularity
# descending.
order_by = None
if not search_query:
order_by = self._get_regional_sort(region, "popularity")
if sort:
if "popularity" in sort:
order_by = self._get_regional_sort(region, "popularity")
elif "trending" in sort:
order_by = self._get_regional_sort(region, "trending")
else:
order_by = [self.DEFAULT_SORTING[name] for name in sort if name in self.DEFAULT_SORTING]
if order_by:
return queryset.sort(*order_by)
return queryset
def _sort_search(request, sq, data):
"""
Sort webapp search based on query + region.
data -- form data.
"""
from mkt.api.base import get_region_from_request
# When querying we want to sort by relevance. If no query is provided,
# i.e. we are only applying filters which don't affect the relevance,
# we sort by popularity descending.
order_by = [] if request.GET.get('q') else ['-popularity']
if data.get('sort'):
region = get_region_from_request(request)
if 'popularity' in data['sort'] and region and not region.adolescent:
# Mature regions sort by their popularity field.
order_by = ['-popularity_%s' % region.id]
else:
order_by = [DEFAULT_SORTING[name] for name in data['sort']
if name in DEFAULT_SORTING]
if order_by:
sq = sq.sort(*order_by)
return sq
def _sort_search(request, sq, data):
"""
Sort webapp search based on query + region.
data -- form data.
"""
from mkt.api.base import get_region_from_request
# When querying we want to sort by relevance. If no query is provided,
# i.e. we are only applying filters which don't affect the relevance,
# we sort by popularity descending.
order_by = [] if request.GET.get('q') else ['-popularity']
if data.get('sort'):
region = get_region_from_request(request)
if 'popularity' in data['sort'] and region and not region.adolescent:
# Mature regions sort by their popularity field.
order_by = ['-popularity_%s' % region.id]
else:
order_by = [
DEFAULT_SORTING[name] for name in data['sort']
if name in DEFAULT_SORTING
]
if order_by:
sq = sq.sort(*order_by)
return sq
def filter_queryset(self, request, queryset, view):
region = get_region_from_request(request)
if region:
return queryset.filter(Bool(must_not=[F("term", region_exclusions=region.id)]))
return queryset
def filter_queryset(self, request, queryset, view):
region = get_region_from_request(request)
search_query = request.GET.get('q')
sort = request.GET.getlist('sort')
# When querying (with `?q=`) we want to sort by relevance. If no query
# is provided and no `?sort` is provided, i.e. we are only applying
# filters which don't affect the relevance, we sort by popularity
# descending.
order_by = None
if not search_query:
order_by = self._get_regional_sort(region, 'popularity')
if sort:
if 'popularity' in sort:
order_by = self._get_regional_sort(region, 'popularity')
elif 'trending' in sort:
order_by = self._get_regional_sort(region, 'trending')
else:
order_by = [
self.DEFAULT_SORTING[name] for name in sort
if name in self.DEFAULT_SORTING
]
if order_by:
return queryset.sort(*order_by)
return queryset
def filter_queryset(self, request, queryset, view):
region = get_region_from_request(request)
if region:
return queryset.filter(
Bool(must_not=[F('term', region_exclusions=region.id)]))
return queryset
def get_app_filter(cls, request, additional_data=None, sq=None,
app_ids=None, no_filter=False):
"""
THE grand, consolidated ES filter for Webapps. By default:
- Excludes non-public apps.
- Excludes disabled apps (whether by reviewer or by developer).
- Excludes based on region exclusions.
- TODO: Excludes based on device and platform support.
additional_data -- an object with more data to allow more filtering.
sq -- if you have an existing search object to filter off of.
app_ids -- if you want to filter by a list of app IDs.
no_filter -- doesn't apply the consumer-side excludes (public/region).
"""
from mkt.api.base import get_region_from_request
from mkt.search.views import name_query
sq = sq or cls.search()
additional_data = additional_data or {}
app_ids = app_ids or []
data = {
'app_type': [],
'author.raw': None,
'category': None, # Slug.
'device': None, # ID.
'gaia': getattr(request, 'GAIA', False),
'is_offline': None,
'manifest_url': '',
'mobile': getattr(request, 'MOBILE', False),
'premium_type': [],
'profile': get_feature_profile(request),
'q': '',
'region': getattr(get_region_from_request(request), 'id', None),
'status': None,
'supported_locales': [],
'tablet': getattr(request, 'TABLET', False),
'tags': '',
}
data.update(additional_data)
# Fields that will be filtered with a term query.
term_fields = ('author.raw', 'device', 'manifest_url', 'status',
'tags')
# Fields that will be filtered with a terms query.
terms_fields = ('category', 'premium_type', 'app_type',
'supported_locales')
# QUERY.
if data['q']:
# Function score for popularity boosting (defaults to multiply).
sq = sq.query(
'function_score',
query=name_query(data['q'].lower()),
functions=[query.SF('field_value_factor', field='boost')])
# MUST.
must = [
F('term', status=amo.STATUS_PUBLIC),
F('term', is_disabled=False),
] if not no_filter else []
for field in term_fields + terms_fields:
# Term filters.
if data[field]:
filter_type = 'term' if field in term_fields else 'terms'
must.append(F(filter_type, **{field: data[field]}))
if not no_filter:
if data['profile']:
# Feature filters.
profile = data['profile']
for k, v in profile.to_kwargs(prefix='features.has_').items():
must.append(F('term', **{k: v}))
if data['mobile'] or data['gaia']:
# Uses flash.
must.append(F('term', uses_flash=False))
if data['is_offline'] is not None:
must.append(F('term', is_offline=data['is_offline']))
# SHOULD.
should = []
if app_ids:
should = [es_filter.Terms(id=list(set(app_ids)))]
sq = sq[0:len(set(app_ids))]
# FILTER.
if must or should:
sq = sq.filter(es_filter.Bool(must=must, should=should))
if data['region'] and not no_filter:
# Region exclusions.
sq = sq.filter(~F('term', region_exclusions=data['region']))
return sq
def filter_queryset(self, request, queryset, view):
q = request.GET.get("q", "").lower()
lang = translation.get_language()
analyzer = self._get_locale_analyzer(lang)
if not q:
return queryset
should = []
rules = [
(query.Match, {"query": q, "boost": 3, "analyzer": "standard"}),
(query.Match, {"query": q, "boost": 4, "type": "phrase", "slop": 1}),
(query.Prefix, {"value": q, "boost": 1.5}),
]
# Only add fuzzy queries if q is a single word. It doesn't make sense
# to do a fuzzy query for multi-word queries.
if " " not in q:
rules.append((query.Fuzzy, {"value": q, "boost": 2, "prefix_length": 1}))
# Apply rules to search on few base fields. Some might not be present
# in every document type / indexes.
for k, v in rules:
for field in ("app_slug", "author", "name", "short_name", "slug", "title", "url_tokenized"):
should.append(k(**{field: v}))
# Exact matches need to be queried against a non-analyzed field. Let's
# do a term query on `name.raw` for an exact match against the item
# name and give it a good boost since this is likely what the user
# wants.
# FIXME: we should also do that on translations and slug/app_slug, but
# we don't store a raw version for them at the moment.
should.append(query.Term(**{"name.raw": {"value": q, "boost": 10}}))
# Do the same for GUID searches.
should.append(query.Term(**{"guid": {"value": q, "boost": 10}}))
# If query is numeric, check if it is an ID.
if q.isnumeric():
should.append(query.Term(**{"id": {"value": q, "boost": 10}}))
if analyzer:
should.append(query.Match(**{"name_l10n_%s" % analyzer: {"query": q, "boost": 2.5}}))
should.append(query.Match(**{"short_name_l10n_%s" % analyzer: {"query": q, "boost": 2.5}}))
# Add searches on the description field.
should.append(query.Match(description={"query": q, "boost": 0.8, "type": "phrase"}))
if analyzer:
desc_field = "description_l10n_%s" % analyzer
desc_analyzer = "%s_analyzer" % analyzer if analyzer in mkt.STEMMER_MAP else analyzer
should.append(
query.Match(**{desc_field: {"query": q, "boost": 0.6, "type": "phrase", "analyzer": desc_analyzer}})
)
# Add searches on tag field.
should.append(query.Term(tags={"value": q}))
if " " not in q:
should.append(query.Fuzzy(tags={"value": q, "prefix_length": 1}))
# The list of functions applied to our `function_score` query.
functions = [query.SF("field_value_factor", field="boost")]
# Add a boost for the preferred region, if it exists.
region = get_region_from_request(request)
if region:
functions.append(
{
"filter": {"term": {"preferred_regions": region.id}},
# TODO: When we upgrade to Elasticsearch 1.4, change this
# to 'weight'.
"boost_factor": 4,
}
)
return queryset.query("function_score", query=query.Bool(should=should), functions=functions)
def filter_queryset(self, request, queryset, view):
q = request.GET.get('q', '').lower()
lang = translation.get_language()
analyzer = self._get_locale_analyzer(lang)
if not q:
return queryset
should = []
rules = [
(query.Match, {'query': q, 'boost': 3, 'analyzer': 'standard'}),
(query.Match, {'query': q, 'boost': 4, 'type': 'phrase',
'slop': 1}),
(query.Prefix, {'value': q, 'boost': 1.5}),
]
# Only add fuzzy queries if q is a single word. It doesn't make sense
# to do a fuzzy query for multi-word queries.
if ' ' not in q:
rules.append(
(query.Fuzzy, {'value': q, 'boost': 2, 'prefix_length': 1}))
# Apply rules to search on few base fields. Some might not be present
# in every document type / indexes.
for k, v in rules:
for field in ('name', 'short_name', 'title', 'app_slug', 'author',
'url_tokenized'):
should.append(k(**{field: v}))
# Exact matches need to be queried against a non-analyzed field. Let's
# do a term query on `name.raw` for an exact match against the item
# name and give it a good boost since this is likely what the user
# wants.
should.append(query.Term(**{'name.raw': {'value': q, 'boost': 10}}))
# Do the same for GUID searches.
should.append(query.Term(**{'guid': {'value': q, 'boost': 10}}))
# If query is numeric, check if it is an ID.
if q.isnumeric():
should.append(query.Term(**{'id': {'value': q, 'boost': 10}}))
if analyzer:
should.append(
query.Match(**{'name_l10n_%s' % analyzer: {'query': q,
'boost': 2.5}}))
should.append(
query.Match(**{'short_name_l10n_%s' % analyzer: {
'query': q,
'boost': 2.5}}))
# Add searches on the description field.
should.append(
query.Match(description={'query': q, 'boost': 0.8,
'type': 'phrase'}))
if analyzer:
desc_field = 'description_l10n_%s' % analyzer
desc_analyzer = ('%s_analyzer' % analyzer
if analyzer in mkt.STEMMER_MAP else analyzer)
should.append(
query.Match(
**{desc_field: {'query': q, 'boost': 0.6, 'type': 'phrase',
'analyzer': desc_analyzer}}))
# Add searches on tag field.
should.append(query.Term(tags={'value': q}))
if ' ' not in q:
should.append(query.Fuzzy(tags={'value': q, 'prefix_length': 1}))
# The list of functions applied to our `function_score` query.
functions = [
query.SF('field_value_factor', field='boost'),
]
# Add a boost for the preferred region, if it exists.
region = get_region_from_request(request)
if region:
functions.append({
'filter': {'term': {'preferred_regions': region.id}},
# TODO: When we upgrade to Elasticsearch 1.4, change this
# to 'weight'.
'boost_factor': 4,
})
return queryset.query('function_score',
query=query.Bool(should=should),
functions=functions)
def filter_queryset(self, request, queryset, view):
q = request.GET.get('q', '').lower()
lang = translation.get_language()
analyzer = self._get_locale_analyzer(lang)
if not q:
return queryset
should = []
rules = [
(query.Match, {
'query': q,
'boost': 3,
'analyzer': 'standard'
}),
(query.Match, {
'query': q,
'boost': 4,
'type': 'phrase',
'slop': 1
}),
(query.Prefix, {
'value': q,
'boost': 1.5
}),
]
# Only add fuzzy queries if q is a single word. It doesn't make sense
# to do a fuzzy query for multi-word queries.
if ' ' not in q:
rules.append((query.Fuzzy, {
'value': q,
'boost': 2,
'prefix_length': 1
}))
# Apply rules to search on few base fields. Some might not be present
# in every document type / indexes.
for k, v in rules:
for field in ('app_slug', 'author', 'name', 'short_name', 'slug',
'title', 'url_tokenized'):
should.append(k(**{field: v}))
# Exact matches need to be queried against a non-analyzed field. Let's
# do a term query on `name.raw` for an exact match against the item
# name and give it a good boost since this is likely what the user
# wants.
# FIXME: we should also do that on translations and slug/app_slug, but
# we don't store a raw version for them at the moment.
should.append(query.Term(**{'name.raw': {'value': q, 'boost': 10}}))
# Do the same for GUID searches.
should.append(query.Term(**{'guid': {'value': q, 'boost': 10}}))
# If query is numeric, check if it is an ID.
if q.isnumeric():
should.append(query.Term(**{'id': {'value': q, 'boost': 10}}))
if analyzer:
should.append(
query.Match(
**{'name_l10n_%s' % analyzer: {
'query': q,
'boost': 2.5
}}))
should.append(
query.Match(**{
'short_name_l10n_%s' % analyzer: {
'query': q,
'boost': 2.5
}
}))
# Add searches on the description field.
should.append(
query.Match(description={
'query': q,
'boost': 0.8,
'type': 'phrase'
}))
if analyzer:
desc_field = 'description_l10n_%s' % analyzer
desc_analyzer = ('%s_analyzer' % analyzer
if analyzer in mkt.STEMMER_MAP else analyzer)
should.append(
query.Match(
**{
desc_field: {
'query': q,
'boost': 0.6,
'type': 'phrase',
'analyzer': desc_analyzer
}
}))
# Add searches on tag field.
should.append(query.Term(tags={'value': q}))
if ' ' not in q:
should.append(query.Fuzzy(tags={'value': q, 'prefix_length': 1}))
# The list of functions applied to our `function_score` query.
functions = [
query.SF('field_value_factor', field='boost'),
]
# Add a boost for the preferred region, if it exists.
region = get_region_from_request(request)
if region:
functions.append({
'filter': {
'term': {
'preferred_regions': region.id
}
},
# TODO: When we upgrade to Elasticsearch 1.4, change this
# to 'weight'.
'boost_factor': 4,
})
return queryset.query('function_score',
query=query.Bool(should=should),
functions=functions)
