def test_or_bool_doesnt_loop_infinitely_issue_96():
q = ~query.Match(f=42) | ~query.Match(f=47)
assert q == query.Bool(should=[
query.Bool(must_not=[query.Match(f=42)]),
query.Bool(must_not=[query.Match(f=47)])
])
def build_first_query(self, args) -> query.Query:
"""
Nếu khớp với 1 sku hoặc 1 phần của 1 sku nào đó, query sẽ trả về sản phẩm có sku đó
Query khớp hoàn toàn với tên sản phẩm, không cho phép khoảng trống cũng như đổi thứ tự
Query sẽ trả về match all nếu text truyền vào rỗng
:param args:
:return: product_es
"""
text_source = args.get('q_source')
search_text = args.get('search_text')
search_text_no_tone = args.get('q')
text_query_condition = query.Bool(
must=[
query.Bool(
should=[query.MatchAll()] if search_text is None else
[
*self.build_sku_match_conditions(text_source),
*self.build_extract_match_text_conditions(search_text, search_text_no_tone, args)
]
),
query.Exists(field="name"),
query.Exists(field="seller")
],
filter=self.get_filter_conditions(args)
)
products_es = self.build_product_es_from_text_query_condition(args, text_query_condition)
# print(json.dumps(products_es.to_dict()))
return products_es
def get_permission(self, user_id, file_ids):
query_conditions = query.Bool(must=[
query.Terms(file_id=file_ids),
query.Bool(should=[
query.Term(owner={
'value': user_id,
'boost': 100
}),
query.Bool(must=[
query.Term(share_mode={
'value': 1,
'boost': 5
}),
query.Term(users_shared={
'value': user_id,
'boost': 5
})
]),
query.Term(share_mode=2)
])
])
file_es = Search() \
.query(query_conditions) \
.source(['owner', 'share_mode', 'editable'])
file_es = file_es[0:1]
print(json.dumps(file_es.to_dict()))
responses = file_es.using(self.es).index(self._index).execute()
return responses
def test_bool_queries_with_only_should_get_concatenated():
q1 = query.Bool(should=[query.Match(f=1), query.Match(f=2)])
q2 = query.Bool(should=[query.Match(f=3), query.Match(f=4)])
assert (q1 | q2) == query.Bool(
should=[query.Match(f=1), query.Match(f=2), query.Match(f=3), query.Match(f=4)]
)
def test_bool_with_different_minimum_should_match_should_not_be_combined():
q1 = query.Q('bool',
minimum_should_match=2,
should=[
query.Q('term', field='aa1'),
query.Q('term', field='aa2'),
query.Q('term', field='aa3'),
query.Q('term', field='aa4')
])
q2 = query.Q('bool',
minimum_should_match=3,
should=[
query.Q('term', field='bb1'),
query.Q('term', field='bb2'),
query.Q('term', field='bb3'),
query.Q('term', field='bb4')
])
q3 = query.Q('bool',
minimum_should_match=4,
should=[
query.Q('term', field='cc1'),
query.Q('term', field='cc2'),
query.Q('term', field='cc3'),
query.Q('term', field='cc4')
])
q4 = q1 | q2
assert q4 == query.Bool(should=[q1, q2])
q5 = q1 | q2 | q3
assert q5 == query.Bool(should=[q1, q2, q3])
def apply_search_query(self, search_query, qs, sort=None):
lang = translation.get_language()
# Our query consist of a number of should clauses. We call the ones
# with the higher boost "primary" for convenience.
primary_should = self.primary_should_rules(search_query, lang)
secondary_should = self.secondary_should_rules(search_query, lang)
# We alter scoring depending on add-on popularity and whether the
# add-on is reviewed & public & non-experimental, and whether or not
# it's in a promoted group with a search boost.
functions = [
query.SF('field_value_factor',
field='average_daily_users',
modifier='log2p'),
query.SF({
'weight':
4.0,
'filter': (Q('term', is_experimental=False)
& Q('terms', status=amo.REVIEWED_STATUSES)
& Q('exists', field='current_version')
& Q('term', is_disabled=False))
}),
]
ranking_bump_groups = amo.utils.sorted_groupby(
PROMOTED_GROUPS, lambda g: g.search_ranking_bump, reverse=True)
for bump, promo_ids in ranking_bump_groups:
if not bump:
continue
functions.append(
query.SF({
'weight':
bump,
'filter':
(Q('terms',
**{'promoted.group_id': [p.id for p in promo_ids]}))
}))
# Assemble everything together
qs = qs.query('function_score',
query=query.Bool(should=primary_should +
secondary_should),
functions=functions)
if sort is None or sort == 'relevance':
# If we are searching by relevancy, rescore the top 10
# (window_size below) results per shard with more expensive rules
# using match_phrase + slop.
rescore_query = self.rescore_rules(search_query, lang)
qs = qs.extra(
rescore={
'window_size': 10,
'query': {
'rescore_query': query.Bool(
should=rescore_query).to_dict()
}
})
return qs
def test_bool_will_append_another_query_with_or():
qb = query.Bool(should=[
query.Match(f='v'),
query.Match(f='v2'),
])
q = query.Match(g=42)
assert (q | qb) == query.Bool(
should=[query.Match(f='v'), query.Match(f='v2'), q])
def test_bool_query_gets_inverted_internally():
q = query.Bool(must_not=[query.Match(f=42)], must=[query.Match(g='v')])
assert ~q == query.Bool(should=[
# negating must
query.Bool(must_not=[query.Match(g='v')]),
# negating must_not
query.Match(f=42),
])
def test_two_bools_are_combined():
q1 = query.Bool(must=[query.MatchAll(), query.Match(f=42)], should=[query.Match(g="v")])
q2 = query.Bool(must=[query.Match(x=42)], should=[query.Match(g="v2")], must_not=[query.Match(title='value')])
q = q1 + q2
assert isinstance(q, query.Bool)
assert q.must == [query.MatchAll(), query.Match(f=42), query.Match(x=42)]
assert q.should == [query.Match(g="v"), query.Match(g="v2")]
assert q.must_not == [query.Match(title='value')]
def apply_search_query(self, search_query, qs, sort=None):
lang = translation.get_language()
analyzer = get_locale_analyzer(lang)
# Our query consist of a number of should clauses. We call the ones
# with the higher boost "primary" for convenience.
primary_should = self.primary_should_rules(search_query, analyzer)
secondary_should = self.secondary_should_rules(search_query, analyzer)
# We alter scoring depending on add-on popularity and whether the
# add-on is reviewed & public & non-experimental.
functions = [
query.SF('field_value_factor',
field='average_daily_users',
modifier='log2p'),
query.SF({
'weight':
4.0,
'filter': (Q('term', is_experimental=False)
& Q('terms', status=amo.REVIEWED_STATUSES)
& Q('exists', field='current_version')
& Q('term', is_disabled=False))
}),
]
if switch_is_active('api-recommendations-priority'):
functions.append(
query.SF({
'weight': 5.0,
'filter': (Q('term', is_recommended=True))
}))
# Assemble everything together
qs = qs.query('function_score',
query=query.Bool(should=primary_should +
secondary_should),
functions=functions)
if sort is None or sort == 'relevance':
# If we are searching by relevancy, rescore the top 10
# (window_size below) results per shard with more expensive rules
# using match_phrase + slop.
rescore_query = self.rescore_rules(search_query, analyzer)
qs = qs.extra(
rescore={
'window_size': 10,
'query': {
'rescore_query': query.Bool(
should=rescore_query).to_dict()
}
})
return qs
def filter_queryset(self, qs):
qs = super(AddonRecommendationView, self).filter_queryset(qs)
guid_param = self.request.GET.get('guid')
taar_enable = self.request.GET.get('recommended', '').lower() == 'true'
guids, self.ab_outcome, self.fallback_reason = (
get_addon_recommendations(guid_param, taar_enable))
results_qs = qs.query(query.Bool(must=[Q('terms', guid=guids)]))
results_qs.execute() # To cache the results.
if results_qs.count() != 4 and is_outcome_recommended(self.ab_outcome):
guids, self.ab_outcome, self.fallback_reason = (
get_addon_recommendations_invalid())
return qs.query(query.Bool(must=[Q('terms', guid=guids)]))
return results_qs
def test_bool_and_other_appends_other_to_must():
q1 = query.Match(f="value1")
qb = query.Bool()
q = qb & q1
assert q is not qb
assert q.must[0] == q1
def test_query_clone():
bool = query.Bool(must=[query.Match(x=42)], should=[query.Match(g="v2")], must_not=[query.Match(title='value')])
bool_clone = bool._clone()
assert bool == bool_clone
assert bool is not bool_clone
assert bool.must[0] is not bool_clone.must[0]
def filter_queryset(self, request, queryset, view):
queries = []
for field in self.fields:
raw_search_param = view.query_params.get(field, '').lower()
wildcard = '*' in raw_search_param or '?' in raw_search_param
search_param = raw_search_param.replace('*', '').replace('?', '')
if not search_param:
# Skip if not given, or just wildcard
continue
# Exact match of sanitized value
queries.append(
Q('term', **{field: {
'value': search_param,
'boost': 10.0
}}))
if wildcard:
# Wildcard search of value as passed
queries.append(
Q('wildcard',
**{field: {
'value': raw_search_param,
'boost': 5.0
}}))
if queries:
queryset = queryset.query(query.Bool(should=queries))
return queryset
def filter_queryset(self, qs):
qs = super(AddonRecommendationView, self).filter_queryset(qs)
guid_param = self.request.GET.get('guid')
taar_enable = self.request.GET.get('recommended', '').lower() == 'true'
guids, self.ab_outcome, self.fallback_reason = (
get_addon_recommendations(guid_param, taar_enable))
return qs.query(query.Bool(must=[Q('terms', guid=guids)]))
def build_second_query(self, args):
"""
Hàm chỉ được gọi khi có search text
:param args:
:param fuzziness:
:return:
"""
fuzzinees = "AUTO" # Use auto fuzzinees for second query
search_text = args.get('search_text')
text_query_condition = query.Bool(
must=[
query.Exists(field="name"),
query.Exists(field="seller")
],
should=[
build_match_query(SEARCH_TEXT_FIELD, search_text, fuzzinees, 2),
build_match_query(SEARCH_TEXT_NO_TONE_FIELD, search_text, fuzzinees)
],
minimum_should_match=1,
filter=self.get_filter_conditions(args)
)
products_es = self.build_product_es_from_text_query_condition(args, text_query_condition)
# print(json.dumps(products_es.to_dict()))
return products_es
def filter_queryset(self, request, queryset, view):
queries = []
for field in self.fields:
raw_search_param = view.query_params.get(field, "").lower()
wildcard = "*" in raw_search_param or "?" in raw_search_param
search_param = raw_search_param.replace("*", "").replace("?", "")
if not search_param:
# Skip if not given, or just wildcard
continue
# Exact match of sanitized value
queries.append(
Q("term", **{field: {
"value": search_param,
"boost": 10.0
}}))
if wildcard:
# Wildcard search of value as passed
queries.append(
Q("wildcard",
**{field: {
"value": raw_search_param,
"boost": 5.0
}}))
if queries:
queryset = queryset.query(query.Bool(should=queries))
return queryset
def content_query(search_term: str, **kwargs) -> Q.DisMax:
"""
Returns the default ONS content query
:param search_term:
:return:
"""
q = Q.DisMax(
queries=[
Q.Bool(
should=[
match(AvailableFields.TITLE_NO_DATES.value.name, search_term, type="boolean", boost=10.0,
minimum_should_match="1<-2 3<80% 5<60%"),
match(AvailableFields.TITLE_NO_STEM.value.name, search_term, type="boolean", boost=10.0,
minimum_should_match="1<-2 3<80% 5<60%"),
multi_match([AvailableFields.TITLE.value.field_name_boosted, AvailableFields.EDITION.value.field_name_boosted], search_term,
type="cross_fields", minimum_should_match="3<80% 5<60%")
]
),
multi_match([AvailableFields.SUMMARY.value.name, AvailableFields.META_DESCRIPTION.value.name], search_term,
type="best_fields", minimum_should_match="75%"),
match(AvailableFields.KEYWORDS.value.name, search_term, type="boolean", operator="AND"),
multi_match([AvailableFields.CDID.value.name, AvailableFields.DATASET_ID.value.name], search_term),
match(AvailableFields.SEARCH_BOOST.value.name, search_term, type="boolean", operator="AND", boost=100.0)
],
**kwargs
)
return q
def filter_queryset(self, qs):
qs = super().filter_queryset(qs)
count = self.paginator.get_page_size(self.request)
self.adzerk_results = get_addons_from_adzerk(count)
ids = list(self.adzerk_results.keys())
group_ids_to_allow = [
group.id for group in PROMOTED_GROUPS
if group.can_be_selected_by_adzerk
]
results_qs = qs.query(
query.Bool(must=[
Q('terms', id=ids),
Q('terms', **{'promoted.group_id': group_ids_to_allow})
]))
results_qs.execute() # To cache the results.
extras = filter_adzerk_results_to_es_results_qs(
self.adzerk_results, results_qs)
if extras:
group_names = '; '.join(
str(group.name) for group in PROMOTED_GROUPS
if group.can_be_selected_by_adzerk)
for id_ in extras:
log.error(
'Addon id [%s] returned from Adzerk, but not in a valid '
'Promoted group [%s]', id_, group_names)
statsd.incr('services.adzerk.elasticsearch_miss', len(extras))
return results_qs
def apply_search_query(self, search_query, qs):
lang = translation.get_language()
analyzer = get_locale_analyzer(lang)
# Our query consist of a number of should clauses. We call the ones
# with the higher boost "primary" for convenience.
primary_should = self.primary_should_rules(search_query, analyzer)
secondary_should = self.secondary_should_rules(search_query, analyzer)
# We alter scoring depending on the "boost" field which is defined in
# the mapping (used to boost public addons higher than the rest) and,
# if the waffle switch is on, whether or an addon is a webextension.
functions = [
query.SF('field_value_factor', field='boost'),
]
if waffle.switch_is_active('boost-webextensions-in-search'):
webext_boost_filter = (Q(
'term', **{'current_version.files.is_webextension': True}) | Q(
'term', **{
'current_version.files.is_mozilla_signed_extension':
True
}))
functions.append(
query.SF({
'weight': WEBEXTENSIONS_WEIGHT,
'filter': webext_boost_filter
}))
# Assemble everything together and return the search "queryset".
return qs.query('function_score',
query=query.Bool(should=primary_should +
secondary_should),
functions=functions)
def filter_queryset(self, request, qs, view):
return qs.query(
query.Bool(
must=[Q('terms', status=amo.REVIEWED_STATUSES),
Q('exists', field='current_version')],
must_not=[Q('term', is_deleted=True),
Q('term', is_disabled=True)]))
def filter_queryset(self, request, qs, view):
search_query = request.GET.get('q', '').lower()
if not search_query:
return qs
lang = translation.get_language()
analyzer = get_locale_analyzer(lang)
# Our query consist of a number of should clauses. We call the ones
# with the higher boost "primary" for convenience.
primary_should = self.primary_should_rules(search_query, analyzer)
secondary_should = self.secondary_should_rules(search_query, analyzer)
# We alter scoring depending on the "boost" field which is defined in
# the mapping (used to boost public addons higher than the rest).
functions = [
query.SF('field_value_factor', field='boost'),
]
# Assemble everything together and return the search "queryset".
return qs.query('function_score',
query=query.Bool(should=primary_should +
secondary_should),
functions=functions)
def filter_queryset(self, request, qs, view):
return qs.query(
query.Bool(filter=[
Q('terms', status=amo.REVIEWED_STATUSES),
Q('exists', field='current_version'),
Q('term', is_disabled=False),
]))
def filter_queryset(self, qs):
qs = super().filter_queryset(qs)
qs = qs.query(query.Bool(filter=[Q('term', is_recommended=True)]))
return (
qs.query('function_score', functions=[query.SF('random_score')])
.sort('_score')
)
def get(self, request, *args, **kwargs):
q = request.GET.get('q')
# Make search.
queries = [
query.Q('match', slug=self._phrase(q)), # Slug.
query.Q('match', type=self._phrase(q)), # Type.
query.Q('match', search_names=self._phrase(q)), # Name.
query.Q('prefix', carrier=q), # Shelf carrier.
query.Q('term', region=q) # Shelf region.
]
sq = query.Bool(should=queries)
# Search.
res = {'apps': [], 'brands': [], 'collections': [], 'shelves': []}
es = Search(using=FeedItemIndexer.get_es(),
index=self.get_feed_element_index())
feed_elements = es.query(sq).execute().hits
if not feed_elements:
return response.Response(res, status=status.HTTP_404_NOT_FOUND)
# Deserialize.
ctx = {
'app_map': self.get_apps(request,
self.get_app_ids_all(feed_elements)),
'request': request
}
for feed_element in feed_elements:
item_type = feed_element.item_type
serializer = self.SERIALIZERS[item_type]
data = serializer(feed_element, context=ctx).data
res[self.PLURAL_TYPES[item_type]].append(data)
# Return.
return response.Response(res, status=status.HTTP_200_OK)
def test_other_and_bool_appends_other_to_must():
q1 = query.Match(f='value1')
qb = query.Bool()
q = q1 & qb
assert q is not qb
assert q.must[0] == q1
def test_bool_and_other_appends_other_to_must():
q1 = query.Match(f='value1')
qb = query.Bool()
q = qb & q1
print(repr(q))
assert q is not qb
assert q.must[0] == q1
def build_sku_query(self, args: dict):
skus = args.get('skus') or []
query_conditions = query.Bool(filter=[
query.Terms(sku=skus)
])
products_es = self.build_product_es_from_text_query_condition(args, query_conditions)
# print(json.dumps(products_es.to_dict()))
return products_es
def build_first_query(self, args):
"""
First version: match all, sort theo score
:return:
"""
brand_search_condition = query.Bool(
must=self.build_first_text_query(args))
brand_es = self.build_brand_es(args, brand_search_condition)
return brand_es
def filter_queryset(self, qs):
qs = super().filter_queryset(qs)
count = self.paginator.get_page_size(self.request)
results = get_addons_from_adzerk(count)
ids = list(results.keys())
results_qs = qs.query(query.Bool(must=[
Q('terms', id=ids),
Q('term', **{'promoted.group_id': VERIFIED_ONE.id})]))
return results_qs
