def test_construction_with_combined_filter_and_exclude_with_exclude_coming_first(
self, test_domain):
q1 = test_domain.get_dao(Person).query.exclude(age=3).filter(
last_name='Doe')
filters1 = q1._owner_dao._build_filters(q1._criteria)
assert filters1 == Bool(must=[Term(last_name='Doe')],
must_not=[Term(age=3)])
def get_filters(data):
filters = []
if 'price_min' in data:
filters.append(Range(price={"gte": data['price_min']}))
if 'price_max' in data:
filters.append(Range(price={"lte": data['price_max']}))
if 'area_min' in data:
filters.append(Range(area={"gte": data['area_min']}))
if 'area_max' in data:
filters.append(Range(area={"lte": data['area_max']}))
if 'rooms' in data:
filters.append(Terms(rooms=data['rooms'].split(',')))
if 'city' in data:
filters.append(Term(city=data['city']))
if 'balcony_type' in data:
filters.append(Terms(balcony_type=data['balcony_type'].split(',')))
if data.get('mortgage', False):
filters.append(Term(mortgage=False))
if data.get('army_mortgage', False):
filters.append(Term(army_mortgage=False))
return filters
def query(self,
q,
kind='term',
field='email.raw',
offset=0,
limit=20,
indices=[INDICES_ALL]):
search = Search(using=self.es, index=indices)
if isinstance(q, dict):
search.update_from_dict(q)
else:
if kind == 'default':
search.query = (Term(**{'email.raw': {
'value': q,
'boost': 3
}}) | Term(**{'username.raw': {
'value': q,
'boost': 2
}}) | Match(username={'query': q}))
else:
cls_ = QUERY_KINDS[kind]
search = search.query(cls_(**{field: q}))
results = search[offset:offset + limit].execute()
return dict(results=results,
total=results.hits.total.value,
hits_relation=results.hits.total.relation,
took=results.took,
timed_out=results.timed_out)
def keywords_query(query, lang):
return Nested(
path='keywords',
query=Bool(
must=[Term(keywords__name=query),
Term(keywords__language=lang)]),
)
def test_construction_with_combined_filter_and_exclude_with_filter_coming_first(
self, test_domain):
q1 = (test_domain.repository_for(Person)._dao.query.filter(
last_name="Doe").exclude(age=3))
filters1 = q1._owner_dao._build_filters(q1._criteria)
assert filters1 == Bool(must=[Term(last_name="Doe")],
must_not=[Term(age=3)])
def get(self, request, *args, **kwargs):
query = self.request.query_params.get('query')
country = self.request.query_params.get('country')
points = self.request.query_params.get('points')
search = Search(index=constants.ES_INDEX)
q = {'should': [], 'filter': []}
if query:
q['should'] = [
Match(variety={
'query': query,
'boost': 3.0
}),
Match(winery={
'query': query,
'boost': 2.0
}),
Match(description={
'query': query,
'boost': 1.0
})
]
q['minimum_should_match'] = 1
search = search.highlight_options(number_of_fragments=0,
pre_tags=['<mark>'],
post_tags=['</mark>'])
search = search.highlight('variety', 'winery', 'description')
if country:
q['filter'].append(Term(country=country))
if points:
q['filter'].append(Term(points=points))
response = search.query('bool', **q).params(size=100).execute()
if response.hits.total.value > 0:
return Response(data=[{
'id':
hit.meta.id,
'country':
hit.country,
'description':
(hit.meta.highlight.description[0] if 'highlight' in hit.meta
and 'description' in hit.meta.highlight else hit.description),
'points':
hit.points,
'price':
hit.price,
'variety': (
hit.meta.highlight.variety[0] if 'highlight' in hit.meta
and 'variety' in hit.meta.highlight else hit.variety),
'winery': (
hit.meta.highlight.winery[0] if 'highlight' in hit.meta
and 'winery' in hit.meta.highlight else hit.winery)
} for hit in response])
else:
return Response(data=[])
def test_create_filter_available_locales(self):
self.assertEqual(create_filter('l', '', SearchWaypoint), None)
self.assertEqual(create_filter('l', 'invalid type', SearchWaypoint),
None)
self.assertEqual(create_filter('l', 'en', SearchWaypoint),
Term(available_locales='en'))
self.assertEqual(create_filter('l', 'en,invalid type', SearchWaypoint),
Term(available_locales='en'))
self.assertEqual(create_filter('l', 'en,fr', SearchWaypoint),
Terms(available_locales=['en', 'fr']))
def test_create_filter_area_ids(self):
self.assertEqual(create_filter('a', '', SearchWaypoint), None)
self.assertEqual(create_filter('a', 'invalid id', SearchWaypoint),
None)
self.assertEqual(create_filter('a', '123', SearchWaypoint),
Term(areas=123))
self.assertEqual(create_filter('a', '123,invalid id', SearchWaypoint),
Term(areas=123))
self.assertEqual(create_filter('a', '123,456', SearchWaypoint),
Terms(areas=[123, 456]))
def test_create_filter_enum(self):
self.assertEqual(create_filter('wtyp', '', SearchWaypoint), None)
self.assertEqual(create_filter('wtyp', 'invalid type', SearchWaypoint),
None)
self.assertEqual(create_filter('wtyp', 'summit', SearchWaypoint),
Term(waypoint_type='summit'))
self.assertEqual(
create_filter('wtyp', 'summit,invalid type', SearchWaypoint),
Term(waypoint_type='summit'))
self.assertEqual(create_filter('wtyp', 'summit,lake', SearchWaypoint),
Terms(waypoint_type=['summit', 'lake']))
def get_filter(self, request, field):
value = request.GET['filter[{}]'.format(field)]
if '.' in field:
path = field.split('.')[0]
return Nested(path=path, query=Term(**{field: value}))
else:
try:
return getattr(self, 'get_{}_filter'.format(field))(value, request)
except AttributeError:
return Term(**{field: value})
def test_create_filter_arrayenum(self):
self.assertEqual(create_filter('wrock', '', SearchWaypoint), None)
self.assertEqual(
create_filter('wrock', 'invalid type', SearchWaypoint), None)
self.assertEqual(create_filter('wrock', 'basalte', SearchWaypoint),
Term(rock_types='basalte'))
self.assertEqual(
create_filter('wrock', 'basalte,invalid type', SearchWaypoint),
Term(rock_types='basalte'))
self.assertEqual(
create_filter('wrock', 'basalte,calcaire', SearchWaypoint),
Terms(rock_types=['basalte', 'calcaire']))
def get_filter(self, request, field):
# Also return activity_manger.id when filtering on owner.id
if field == 'owner.id':
value = request.GET['filter[{}]'.format(field)]
return Q(
Nested(path='owner', query=Term(**{field: value}))
| Nested(path='promoter', query=Term(promoter__id=value))
| Nested(path='activity_owners',
query=Term(activity_owners__id=value))
| Nested(path='activity_manager',
query=Term(activity_manager__id=value)))
return super(InitiativeSearchFilter, self).get_filter(request, field)
def get_default_filters(self, request):
fields = super(InitiativeSearchFilter, self).get_filter_fields(request)
permission = 'initiatives.api_read_initiative'
if not request.user.has_perm(permission):
filters = [Term(owner_id=request.user.id)]
if 'owner.id' not in fields:
filters.append(Term(status='approved'))
return filters
elif 'owner.id' in fields and request.user.is_authenticated:
value = request.user.pk
return [
Nested(path='owner', query=Term(owner__id=value))
| Nested(path='promoter', query=Term(promoter__id=value))
| Nested(path='activity_manager',
query=Term(activity_manager__id=value))
| Nested(path='activity_owners',
query=Term(activity_owners__id=value))
| Term(status='approved')
]
else:
return [Term(status='approved')]
def test_construction_of_filter_with_two_attributes(self, test_domain):
q1 = test_domain.repository_for(Person)._dao.query.filter(
first_name="Jane", last_name="Doe")
q2 = (test_domain.repository_for(Person)._dao.query.filter(
first_name="Jane").filter(last_name="Doe"))
filters1 = q1._owner_dao._build_filters(q1._criteria)
filters2 = q2._owner_dao._build_filters(q2._criteria)
assert filters1 == Bool(
must=[Term(first_name="Jane"),
Term(last_name="Doe")])
assert filters2 == Bool(
must=[Term(first_name="Jane"),
Term(last_name="Doe")])
def test_construction_of_filter_with_two_attributes(self, test_domain):
q1 = test_domain.get_dao(Person).query.filter(first_name='Jane',
last_name='Doe')
q2 = test_domain.get_dao(Person).query.filter(
first_name='Jane').filter(last_name='Doe')
filters1 = q1._owner_dao._build_filters(q1._criteria)
filters2 = q2._owner_dao._build_filters(q2._criteria)
assert filters1 == Bool(
must=[Term(first_name='Jane'),
Term(last_name='Doe')])
assert filters2 == Bool(
must=[Term(first_name='Jane'),
Term(last_name='Doe')])
def get_base_query(self, request, validated_data):
"""
Get the base query.
This is overridden to exclude UPDATE history items and interactions
without export countries.
"""
base_query = super().get_base_query(request, validated_data)
is_relevant_interaction = Term(were_countries_discussed=True)
is_relevant_history_entry = Term(
_document_type=ExportCountryHistoryApp.name)
return base_query.filter(is_relevant_interaction
| is_relevant_history_entry)
def test_construction_of_filter_with_two_negated_attributes(
self, test_domain):
q1 = test_domain.get_dao(Person).query.exclude(first_name="Jane",
last_name="Doe")
q2 = (test_domain.get_dao(Person).query.exclude(
first_name="Jane").exclude(last_name="Doe"))
filters1 = q1._owner_dao._build_filters(q1._criteria)
filters2 = q2._owner_dao._build_filters(q2._criteria)
assert filters1 == Bool(
must_not=[Term(first_name="Jane"),
Term(last_name="Doe")])
assert filters2 == Bool(
must_not=[Term(first_name="Jane"),
Term(last_name="Doe")])
def get_queryset_posts(self):
"""Search in posts, and remove result if the forum is not allowed for the user or if the message is invisible.
Score is modified if:
+ post is the first one in a topic;
+ post is marked as "useful";
+ post has a like/dislike ratio above (has more likes than dislikes) or below (the other way around) 1.0.
"""
query = Match(_type='post') \
& Terms(forum_pk=self.authorized_forums) \
& Term(is_visible=True) \
& MultiMatch(query=self.search_query, fields=['text_html'])
functions_score = [
{'filter': Match(position=1), 'weight': settings.ZDS_APP['search']['boosts']['post']['if_first']},
{'filter': Match(is_useful=True), 'weight': settings.ZDS_APP['search']['boosts']['post']['if_useful']},
{
'filter': Range(like_dislike_ratio={'gt': 1}),
'weight': settings.ZDS_APP['search']['boosts']['post']['ld_ratio_above_1']
},
{
'filter': Range(like_dislike_ratio={'lt': 1}),
'weight': settings.ZDS_APP['search']['boosts']['post']['ld_ratio_below_1']
}
]
scored_query = FunctionScore(query=query, boost_mode='multiply', functions=functions_score)
return scored_query
def get_queryset_posts(self):
"""Search in posts, and remove result if the forum is not allowed for the user or if the message is invisible.
Score is modified if:
+ post is the first one in a topic;
+ post is marked as "useful";
+ post has a like/dislike ratio above (has more likes than dislikes) or below (the other way around) 1.0.
"""
query = (
Match(_type="post")
& Terms(forum_pk=self.authorized_forums)
& Term(is_visible=True)
& MultiMatch(query=self.search_query, fields=["text_html"])
)
functions_score = [
{"filter": Match(position=1), "weight": settings.ZDS_APP["search"]["boosts"]["post"]["if_first"]},
{"filter": Match(is_useful=True), "weight": settings.ZDS_APP["search"]["boosts"]["post"]["if_useful"]},
{
"filter": Range(like_dislike_ratio={"gt": 1}),
"weight": settings.ZDS_APP["search"]["boosts"]["post"]["ld_ratio_above_1"],
},
{
"filter": Range(like_dislike_ratio={"lt": 1}),
"weight": settings.ZDS_APP["search"]["boosts"]["post"]["ld_ratio_below_1"],
},
]
scored_query = FunctionScore(query=query, boost_mode="multiply", functions=functions_score)
return scored_query
def make_filter(type, agg_type, agg_key):
if type == "text":
# filters with '/' might be leading to books. also, very unlikely they'll match an false positives
reg = re.escape(agg_key) + (".*" if "/" in agg_key else "/.*")
return Regexp(path=reg)
elif type == "sheet":
return Term(**{agg_type: agg_key})
def etl(index='cf_rfem_hist_price',
start_date='2018-12-26',
end_date='2019-03-25',
symbol='rfem'):
ESLowLevelClientByConnection.get_instance()
search = Search(index=index, using='high_level_client')[0:100]
search.query = Q(
Bool(must=[
Range(date={
'gte': '2018-12-26',
'lte': '2019-03-25'
}),
Term(symbol='rfem')
]))
aggs = A(
DateHistogram(field='date',
interval='1d',
format='yyyy-MM-dd',
min_doc_count=1))
response = search.execute()
hits = response['hits']
hits = hits['hits']
XX = []
for hit in hits:
X = []
X.append(hit['_source']['changeOverTime'])
X.append(hit['_source']['changePercent'])
X.append(hit['_source']['volume'])
XX.append(X)
return (XX)
def query(self, search, query):
"""
Manipulates the query to support nested queries and a custom rank for pages.
If `self.projects` was given, we use it to filter the documents that
match the same project and version.
"""
search = search.highlight_options(**self._highlight_options)
search = search.source(excludes=self.excludes)
queries = self._get_queries(
query=query,
fields=self.fields,
)
sections_nested_query = self._get_nested_query(
query=query,
path='sections',
fields=self._section_fields,
)
domains_nested_query = self._get_nested_query(
query=query,
path='domains',
fields=self._domain_fields,
)
queries.extend([sections_nested_query, domains_nested_query])
bool_query = Bool(should=queries)
if self.projects:
versions_query = [
Bool(must=[
Term(project={'value': project}),
Term(version={'value': version}),
]) for project, version in self.projects.items()
]
bool_query = Bool(must=[bool_query, Bool(should=versions_query)])
final_query = FunctionScore(
query=bool_query,
script_score=self._get_script_score(),
)
search = search.query(final_query)
return search
def _build_global_permission_subqueries(permission_filters_by_entity):
for entity, filter_args in permission_filters_by_entity.items():
query = Term(_document_type=entity)
entity_condition = _build_entity_permission_query(filter_args)
if entity_condition is not None:
query &= entity_condition
yield query
def get_filters_for_stats(self, data):
filters = []
if 'rooms' in data:
filters.append(Terms(rooms=data['rooms'].split(',')))
if 'city' in data:
filters.append(Term(city=data['city']))
return filters
def get_filters(self, request):
filters = super(ActivitySearchFilter, self).get_filters(request)
regex = re.compile('^filter\[segment\.(?P<type>[\w\-]+)\]$')
for key, value in list(request.GET.items()):
matches = regex.match(key)
if matches:
filters.append(
Nested(
path='segments',
query=Term(
segments__type=matches.groupdict()['type']
) & Term(
segments__id=value
)
)
)
return filters
def test_build_query_sort_outing(self):
params = {'act': 'skitouring'}
meta_params = {'limit': 20, 'offset': 40}
query = build_query(params, meta_params, 'o')
expected_query = create_search('o'). \
filter(Term(activities='skitouring')).\
fields([]).\
sort({'date_end': {'order': 'desc'}}, {'id': {'order': 'desc'}}).\
extra(from_=40, size=20)
self.assertQueryEqual(query, expected_query)
def test_get_elasticsearch_query(app, db, es, test_users):
with db.session.begin_nested():
actor = RecordUserActor(name='test',
originator=test_users.u1,
path='/creator-id')
db.session.add(actor)
assert Term(_invenio_explicit_acls__user=test_users.u3.id) == \
actor.get_elasticsearch_query(test_users.u3, {'system_roles': [authenticated_user]})
assert actor.get_elasticsearch_query(AnonymousUser(),
{'system_roles': [any_user]}) is None
def search_tweets(self, query, first_tweet = 0, num_tweets = 10):
'''
Es igual que el método anterior, solo que el resultado de la query es parseado
(convierte cada documento en forma de diccionario, en instancias de la clase Tweet)
Además, se refinará la busqueda sobre elasticsearch de modo que solo se busquen aquellos
documentos cuyo proveedor es twitter.
'''
query &= Term(provider = 'twitter')
docs = self._search(query, first_doc = first_tweet, num_docs = num_tweets)
def parse(doc):
source = doc['_source']
# Comprobamos que el proveedor del documento es twitter
provider = source['provider']
if not provider == 'twitter':
raise Exception()
id = source['post_id']
text = next(iter(source['body'].values()))
is_retweet = source['is_retweet']
is_reply = source['is_reply']
post_type = 'retweet' if is_retweet else ('reply' if is_reply else 'original')
num_retweets = source['retweet_count']
retweet_id = source['retweet_id'] if post_type == 'retweet' else None
reply_id = source['retweet_id'] if post_type == 'reply' else None
author = None
result = match('^(\d{4})\-(\d{1,2})\-(\d{1,2})T(\d{1,2})\:(\d{1,2})\+.*$', source['published_at'])
if not result:
raise Exception()
timestamp = datetime(*[int(strnum) for strnum in result.groups()]).strftime('%s')
user_data = source['user']
num_followers = user_data['followers_count']
num_friends = user_data['friends_count']
screen_name = user_data['screenname']
author = TwitterUser(screen_name, num_followers, num_friends)
tweet = Tweet(author, id, text, post_type, num_retweets, timestamp, retweet_id, reply_id)
return tweet
posts = []
for doc in docs:
try:
post = parse(doc)
posts.append(post)
except:
pass
return posts
def get_queryset(self):
"""
Manipulate the returned queryset by adding a filter for tenant using the tenant linked
to the current logged in user (received via custom middleware).
"""
user = get_current_user()
if user and user.is_authenticated():
qs = super(ElasticTenantManager,
self).get_queryset().filter(tenant_id=user.tenant_id)
return qs.elasticsearch_query(Term(tenant_id=user.tenant_id))
else:
return super(ElasticTenantManager, self).get_queryset()
def _get_projects_query(self):
"""
Get filter by projects query.
If it's a dict, filter by project and version,
if it's a list filter by project.
"""
if not self.projects:
return None
if isinstance(self.projects, dict):
versions_query = [
Bool(filter=[Term(project=project), Term(version=version)])
for project, version in self.projects.items()
]
return Bool(should=versions_query)
if isinstance(self.projects, list):
return Bool(filter=Terms(project=self.projects))
raise ValueError('projects must be a list or a dict!')