Example #1
0
    def filter_queryset(self, request, queryset, view):

        region = get_region_from_request(request)
        search_query = request.GET.get("q")
        sort = request.GET.getlist("sort")

        # When querying (with `?q=`) we want to sort by relevance. If no query
        # is provided and no `?sort` is provided, i.e. we are only applying
        # filters which don't affect the relevance, we sort by popularity
        # descending.
        order_by = None
        if not search_query:
            order_by = self._get_regional_sort(region, "popularity")

        if sort:
            if "popularity" in sort:
                order_by = self._get_regional_sort(region, "popularity")
            elif "trending" in sort:
                order_by = self._get_regional_sort(region, "trending")
            else:
                order_by = [self.DEFAULT_SORTING[name] for name in sort if name in self.DEFAULT_SORTING]

        if order_by:
            return queryset.sort(*order_by)

        return queryset
Example #2
0
def _sort_search(request, sq, data):
    """
    Sort webapp search based on query + region.

    data -- form data.
    """
    from mkt.api.base import get_region_from_request

    # When querying we want to sort by relevance. If no query is provided,
    # i.e. we are only applying filters which don't affect the relevance,
    # we sort by popularity descending.
    order_by = [] if request.GET.get('q') else ['-popularity']

    if data.get('sort'):
        region = get_region_from_request(request)
        if 'popularity' in data['sort'] and region and not region.adolescent:
            # Mature regions sort by their popularity field.
            order_by = ['-popularity_%s' % region.id]
        else:
            order_by = [DEFAULT_SORTING[name] for name in data['sort']
                        if name in DEFAULT_SORTING]

    if order_by:
        sq = sq.sort(*order_by)
    return sq
Example #3
0
def _sort_search(request, sq, data):
    """
    Sort webapp search based on query + region.

    data -- form data.
    """
    from mkt.api.base import get_region_from_request

    # When querying we want to sort by relevance. If no query is provided,
    # i.e. we are only applying filters which don't affect the relevance,
    # we sort by popularity descending.
    order_by = [] if request.GET.get('q') else ['-popularity']

    if data.get('sort'):
        region = get_region_from_request(request)
        if 'popularity' in data['sort'] and region and not region.adolescent:
            # Mature regions sort by their popularity field.
            order_by = ['-popularity_%s' % region.id]
        else:
            order_by = [
                DEFAULT_SORTING[name] for name in data['sort']
                if name in DEFAULT_SORTING
            ]

    if order_by:
        sq = sq.sort(*order_by)
    return sq
Example #4
0
    def filter_queryset(self, request, queryset, view):

        region = get_region_from_request(request)
        if region:
            return queryset.filter(Bool(must_not=[F("term", region_exclusions=region.id)]))

        return queryset
Example #5
0
    def filter_queryset(self, request, queryset, view):

        region = get_region_from_request(request)
        search_query = request.GET.get('q')
        sort = request.GET.getlist('sort')

        # When querying (with `?q=`) we want to sort by relevance. If no query
        # is provided and no `?sort` is provided, i.e. we are only applying
        # filters which don't affect the relevance, we sort by popularity
        # descending.
        order_by = None
        if not search_query:
            order_by = self._get_regional_sort(region, 'popularity')

        if sort:
            if 'popularity' in sort:
                order_by = self._get_regional_sort(region, 'popularity')
            elif 'trending' in sort:
                order_by = self._get_regional_sort(region, 'trending')
            else:
                order_by = [
                    self.DEFAULT_SORTING[name] for name in sort
                    if name in self.DEFAULT_SORTING
                ]

        if order_by:
            return queryset.sort(*order_by)

        return queryset
Example #6
0
    def filter_queryset(self, request, queryset, view):

        region = get_region_from_request(request)
        if region:
            return queryset.filter(
                Bool(must_not=[F('term', region_exclusions=region.id)]))

        return queryset
Example #7
0
    def get_app_filter(cls, request, additional_data=None, sq=None,
                       app_ids=None, no_filter=False):
        """
        THE grand, consolidated ES filter for Webapps. By default:
        - Excludes non-public apps.
        - Excludes disabled apps (whether by reviewer or by developer).
        - Excludes based on region exclusions.
        - TODO: Excludes based on device and platform support.

        additional_data -- an object with more data to allow more filtering.
        sq -- if you have an existing search object to filter off of.
        app_ids -- if you want to filter by a list of app IDs.
        no_filter -- doesn't apply the consumer-side excludes (public/region).
        """
        from mkt.api.base import get_region_from_request
        from mkt.search.views import name_query

        sq = sq or cls.search()
        additional_data = additional_data or {}
        app_ids = app_ids or []

        data = {
            'app_type': [],
            'author.raw': None,
            'category': None,  # Slug.
            'device': None,  # ID.
            'gaia': getattr(request, 'GAIA', False),
            'is_offline': None,
            'manifest_url': '',
            'mobile': getattr(request, 'MOBILE', False),
            'premium_type': [],
            'profile': get_feature_profile(request),
            'q': '',
            'region': getattr(get_region_from_request(request), 'id', None),
            'status': None,
            'supported_locales': [],
            'tablet': getattr(request, 'TABLET', False),
            'tags': '',
        }
        data.update(additional_data)

        # Fields that will be filtered with a term query.
        term_fields = ('author.raw', 'device', 'manifest_url', 'status',
                       'tags')
        # Fields that will be filtered with a terms query.
        terms_fields = ('category', 'premium_type', 'app_type',
                        'supported_locales')

        # QUERY.
        if data['q']:
            # Function score for popularity boosting (defaults to multiply).
            sq = sq.query(
                'function_score',
                query=name_query(data['q'].lower()),
                functions=[query.SF('field_value_factor', field='boost')])

        # MUST.
        must = [
            F('term', status=amo.STATUS_PUBLIC),
            F('term', is_disabled=False),
        ] if not no_filter else []

        for field in term_fields + terms_fields:
            # Term filters.
            if data[field]:
                filter_type = 'term' if field in term_fields else 'terms'
                must.append(F(filter_type, **{field: data[field]}))

        if not no_filter:
            if data['profile']:
                # Feature filters.
                profile = data['profile']
                for k, v in profile.to_kwargs(prefix='features.has_').items():
                    must.append(F('term', **{k: v}))
            if data['mobile'] or data['gaia']:
                # Uses flash.
                must.append(F('term', uses_flash=False))
            if data['is_offline'] is not None:
                must.append(F('term', is_offline=data['is_offline']))

        # SHOULD.
        should = []
        if app_ids:
            should = [es_filter.Terms(id=list(set(app_ids)))]
            sq = sq[0:len(set(app_ids))]

        # FILTER.
        if must or should:
            sq = sq.filter(es_filter.Bool(must=must, should=should))

        if data['region'] and not no_filter:
            # Region exclusions.
            sq = sq.filter(~F('term', region_exclusions=data['region']))

        return sq
Example #8
0
    def filter_queryset(self, request, queryset, view):

        q = request.GET.get("q", "").lower()
        lang = translation.get_language()
        analyzer = self._get_locale_analyzer(lang)

        if not q:
            return queryset

        should = []
        rules = [
            (query.Match, {"query": q, "boost": 3, "analyzer": "standard"}),
            (query.Match, {"query": q, "boost": 4, "type": "phrase", "slop": 1}),
            (query.Prefix, {"value": q, "boost": 1.5}),
        ]

        # Only add fuzzy queries if q is a single word. It doesn't make sense
        # to do a fuzzy query for multi-word queries.
        if " " not in q:
            rules.append((query.Fuzzy, {"value": q, "boost": 2, "prefix_length": 1}))

        # Apply rules to search on few base fields. Some might not be present
        # in every document type / indexes.
        for k, v in rules:
            for field in ("app_slug", "author", "name", "short_name", "slug", "title", "url_tokenized"):
                should.append(k(**{field: v}))

        # Exact matches need to be queried against a non-analyzed field. Let's
        # do a term query on `name.raw` for an exact match against the item
        # name and give it a good boost since this is likely what the user
        # wants.
        # FIXME: we should also do that on translations and slug/app_slug, but
        # we don't store a raw version for them at the moment.
        should.append(query.Term(**{"name.raw": {"value": q, "boost": 10}}))
        # Do the same for GUID searches.
        should.append(query.Term(**{"guid": {"value": q, "boost": 10}}))
        # If query is numeric, check if it is an ID.
        if q.isnumeric():
            should.append(query.Term(**{"id": {"value": q, "boost": 10}}))

        if analyzer:
            should.append(query.Match(**{"name_l10n_%s" % analyzer: {"query": q, "boost": 2.5}}))
            should.append(query.Match(**{"short_name_l10n_%s" % analyzer: {"query": q, "boost": 2.5}}))

        # Add searches on the description field.
        should.append(query.Match(description={"query": q, "boost": 0.8, "type": "phrase"}))

        if analyzer:
            desc_field = "description_l10n_%s" % analyzer
            desc_analyzer = "%s_analyzer" % analyzer if analyzer in mkt.STEMMER_MAP else analyzer
            should.append(
                query.Match(**{desc_field: {"query": q, "boost": 0.6, "type": "phrase", "analyzer": desc_analyzer}})
            )

        # Add searches on tag field.
        should.append(query.Term(tags={"value": q}))
        if " " not in q:
            should.append(query.Fuzzy(tags={"value": q, "prefix_length": 1}))

        # The list of functions applied to our `function_score` query.
        functions = [query.SF("field_value_factor", field="boost")]

        # Add a boost for the preferred region, if it exists.
        region = get_region_from_request(request)
        if region:
            functions.append(
                {
                    "filter": {"term": {"preferred_regions": region.id}},
                    # TODO: When we upgrade to Elasticsearch 1.4, change this
                    # to 'weight'.
                    "boost_factor": 4,
                }
            )

        return queryset.query("function_score", query=query.Bool(should=should), functions=functions)
Example #9
0
    def filter_queryset(self, request, queryset, view):

        q = request.GET.get('q', '').lower()
        lang = translation.get_language()
        analyzer = self._get_locale_analyzer(lang)

        if not q:
            return queryset

        should = []
        rules = [
            (query.Match, {'query': q, 'boost': 3, 'analyzer': 'standard'}),
            (query.Match, {'query': q, 'boost': 4, 'type': 'phrase',
                           'slop': 1}),
            (query.Prefix, {'value': q, 'boost': 1.5}),
        ]

        # Only add fuzzy queries if q is a single word. It doesn't make sense
        # to do a fuzzy query for multi-word queries.
        if ' ' not in q:
            rules.append(
                (query.Fuzzy, {'value': q, 'boost': 2, 'prefix_length': 1}))

        # Apply rules to search on few base fields. Some might not be present
        # in every document type / indexes.
        for k, v in rules:
            for field in ('name', 'short_name', 'title', 'app_slug', 'author',
                          'url_tokenized'):
                should.append(k(**{field: v}))

        # Exact matches need to be queried against a non-analyzed field. Let's
        # do a term query on `name.raw` for an exact match against the item
        # name and give it a good boost since this is likely what the user
        # wants.
        should.append(query.Term(**{'name.raw': {'value': q, 'boost': 10}}))
        # Do the same for GUID searches.
        should.append(query.Term(**{'guid': {'value': q, 'boost': 10}}))
        # If query is numeric, check if it is an ID.
        if q.isnumeric():
            should.append(query.Term(**{'id': {'value': q, 'boost': 10}}))

        if analyzer:
            should.append(
                query.Match(**{'name_l10n_%s' % analyzer: {'query': q,
                                                           'boost': 2.5}}))
            should.append(
                query.Match(**{'short_name_l10n_%s' % analyzer: {
                    'query': q,
                    'boost': 2.5}}))

        # Add searches on the description field.
        should.append(
            query.Match(description={'query': q, 'boost': 0.8,
                                     'type': 'phrase'}))

        if analyzer:
            desc_field = 'description_l10n_%s' % analyzer
            desc_analyzer = ('%s_analyzer' % analyzer
                             if analyzer in mkt.STEMMER_MAP else analyzer)
            should.append(
                query.Match(
                    **{desc_field: {'query': q, 'boost': 0.6, 'type': 'phrase',
                                    'analyzer': desc_analyzer}}))

        # Add searches on tag field.
        should.append(query.Term(tags={'value': q}))
        if ' ' not in q:
            should.append(query.Fuzzy(tags={'value': q, 'prefix_length': 1}))

        # The list of functions applied to our `function_score` query.
        functions = [
            query.SF('field_value_factor', field='boost'),
        ]

        # Add a boost for the preferred region, if it exists.
        region = get_region_from_request(request)
        if region:
            functions.append({
                'filter': {'term': {'preferred_regions': region.id}},
                # TODO: When we upgrade to Elasticsearch 1.4, change this
                # to 'weight'.
                'boost_factor': 4,
            })

        return queryset.query('function_score',
                              query=query.Bool(should=should),
                              functions=functions)
Example #10
0
    def filter_queryset(self, request, queryset, view):

        q = request.GET.get('q', '').lower()
        lang = translation.get_language()
        analyzer = self._get_locale_analyzer(lang)

        if not q:
            return queryset

        should = []
        rules = [
            (query.Match, {
                'query': q,
                'boost': 3,
                'analyzer': 'standard'
            }),
            (query.Match, {
                'query': q,
                'boost': 4,
                'type': 'phrase',
                'slop': 1
            }),
            (query.Prefix, {
                'value': q,
                'boost': 1.5
            }),
        ]

        # Only add fuzzy queries if q is a single word. It doesn't make sense
        # to do a fuzzy query for multi-word queries.
        if ' ' not in q:
            rules.append((query.Fuzzy, {
                'value': q,
                'boost': 2,
                'prefix_length': 1
            }))

        # Apply rules to search on few base fields. Some might not be present
        # in every document type / indexes.
        for k, v in rules:
            for field in ('app_slug', 'author', 'name', 'short_name', 'slug',
                          'title', 'url_tokenized'):
                should.append(k(**{field: v}))

        # Exact matches need to be queried against a non-analyzed field. Let's
        # do a term query on `name.raw` for an exact match against the item
        # name and give it a good boost since this is likely what the user
        # wants.
        # FIXME: we should also do that on translations and slug/app_slug, but
        # we don't store a raw version for them at the moment.
        should.append(query.Term(**{'name.raw': {'value': q, 'boost': 10}}))
        # Do the same for GUID searches.
        should.append(query.Term(**{'guid': {'value': q, 'boost': 10}}))
        # If query is numeric, check if it is an ID.
        if q.isnumeric():
            should.append(query.Term(**{'id': {'value': q, 'boost': 10}}))

        if analyzer:
            should.append(
                query.Match(
                    **{'name_l10n_%s' % analyzer: {
                        'query': q,
                        'boost': 2.5
                    }}))
            should.append(
                query.Match(**{
                    'short_name_l10n_%s' % analyzer: {
                        'query': q,
                        'boost': 2.5
                    }
                }))

        # Add searches on the description field.
        should.append(
            query.Match(description={
                'query': q,
                'boost': 0.8,
                'type': 'phrase'
            }))

        if analyzer:
            desc_field = 'description_l10n_%s' % analyzer
            desc_analyzer = ('%s_analyzer' % analyzer
                             if analyzer in mkt.STEMMER_MAP else analyzer)
            should.append(
                query.Match(
                    **{
                        desc_field: {
                            'query': q,
                            'boost': 0.6,
                            'type': 'phrase',
                            'analyzer': desc_analyzer
                        }
                    }))

        # Add searches on tag field.
        should.append(query.Term(tags={'value': q}))
        if ' ' not in q:
            should.append(query.Fuzzy(tags={'value': q, 'prefix_length': 1}))

        # The list of functions applied to our `function_score` query.
        functions = [
            query.SF('field_value_factor', field='boost'),
        ]

        # Add a boost for the preferred region, if it exists.
        region = get_region_from_request(request)
        if region:
            functions.append({
                'filter': {
                    'term': {
                        'preferred_regions': region.id
                    }
                },
                # TODO: When we upgrade to Elasticsearch 1.4, change this
                # to 'weight'.
                'boost_factor': 4,
            })

        return queryset.query('function_score',
                              query=query.Bool(should=should),
                              functions=functions)