def test_prefill_search_navigation(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_query(search)
assert search.navigation_ids == []
search = SearchQuery()
req = ParsedRequest()
req.args = {'navigation': ''}
service.prefill_search_query(search, req)
assert search.navigation_ids == []
search = SearchQuery()
req = ParsedRequest()
req.args = {'navigation': '{},{},{}'.format(NAV_1, NAV_2, NAV_3)}
service.prefill_search_query(search, req)
assert search.navigation_ids == [str(NAV_1), str(NAV_2), str(NAV_3)]
search = SearchQuery()
req = ParsedRequest()
req.args = {'navigation': [str(NAV_1), str(NAV_2), str(NAV_3)]}
service.prefill_search_query(search, req)
assert search.navigation_ids == [str(NAV_1), str(NAV_2), str(NAV_3)]
search = SearchQuery()
req = ParsedRequest()
req.args = {'navigation': {'test': NAV_1}}
with raises(BadParameterValueError):
service.prefill_search_query(search, req)
def test_prefill_search_args(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_args(search)
assert search.args == {}
assert search.projections == {}
assert search.req is None
search = SearchQuery()
req = ParsedRequest()
req.args = {'test': 'one'}
service.prefill_search_args(search, req)
assert search.args == {'test': 'one'}
assert search.projections == {}
assert search.req == req
search = SearchQuery()
req = ParsedRequest()
req.args = ImmutableMultiDict([('foo', 'bar'), ('name', 'test')])
service.prefill_search_args(search, req)
assert search.args == {'foo': 'bar', 'name': 'test'}
assert search.projections == {}
assert search.req == req
search = SearchQuery()
req = ParsedRequest()
req.projection = {'service': 1}
service.prefill_search_args(search, req)
assert search.args == {}
assert search.projections == {'service': 1}
assert search.req == req
def test_prefill_search_user(client, app):
with app.test_request_context():
session['user'] = None
search = SearchQuery()
service.prefill_search_query(search)
assert search.user is None
session['user'] = ADMIN_USER_ID
search = SearchQuery()
service.prefill_search_query(search)
assert search.user.get('_id') == ADMIN_USER_ID
assert search.is_admin is True
session['user'] = PUBLIC_USER_ID
search = SearchQuery()
service.prefill_search_query(search)
assert search.user.get('_id') == PUBLIC_USER_ID
assert search.is_admin is False
session['user'] = ADMIN_USER_ID
search = SearchQuery()
req = ParsedRequest()
req.args = {'user': TEST_USER_ID}
service.prefill_search_query(search, req)
assert search.user.get('_id') == TEST_USER_ID
assert search.is_admin is False
def get_saved_items_count(self):
search = SearchQuery()
search.query = _agenda_query()
self.prefill_search_query(search)
self.apply_filters(search)
set_saved_items_query(search.query, str(search.user['_id']))
cursor = self.get_items_by_query(search.query, size=0)
return cursor.count() if cursor else 0
def test_prefill_search_section(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_section(search)
assert search.section == service.section
search = SearchQuery()
service.section = 'test'
service.prefill_search_section(search)
assert search.section == 'test'
def test_prefill_search_lookup(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_query(search)
assert search.lookup == {}
search = SearchQuery()
service.prefill_search_query(search, lookup={})
assert search.lookup == {}
search = SearchQuery()
service.prefill_search_query(search, lookup={'foo': 'bar'})
assert search.lookup == {'foo': 'bar'}
def get_navigation_story_count(self, navigations, section, company, user):
"""Get story count by navigation"""
search = SearchQuery()
self.prefill_search_args(search)
self.prefill_search_items(search)
search.section = section
search.user = user
search.company = company
self.apply_section_filter(search)
aggs = {'navigations': {'filters': {'filters': {}}}}
for navigation in navigations:
navigation_id = navigation.get('_id')
products = get_products_by_navigation(navigation_id) or []
navigation_filter = {
'bool': {
'should': [],
'minimum_should_match': 1
}
}
for product in products:
if product.get('query'):
navigation_filter['bool']['should'].append(
query_string(product.get('query')))
if navigation_filter['bool']['should']:
aggs['navigations']['filters']['filters'][str(
navigation_id)] = navigation_filter
source = {'query': search.query, 'aggs': aggs, 'size': 0}
req = ParsedRequest()
req.args = {'source': json.dumps(source)}
try:
results = self.internal_get(req, None)
buckets = results.hits['aggregations']['navigations']['buckets']
for navigation in navigations:
navigation_id = navigation.get('_id')
doc_count = buckets.get(str(navigation_id),
{}).get('doc_count', 0)
if doc_count > 0:
navigation['story_count'] = doc_count
except Exception as exc:
logger.error(
'Error in get_navigation_story_count for query: {}'.format(
json.dumps(source)),
exc,
exc_info=True)
def get_items(self,
item_ids,
size=None,
aggregations=None,
apply_permissions=False):
search = SearchQuery()
try:
search.query = {
'bool': {
'must_not': [
{
'term': {
'type': 'composite'
}
},
],
'must': [{
'terms': {
'_id': item_ids
}
}],
'should': [],
}
}
if apply_permissions:
self.prefill_search_query(search)
self.validate_request(search)
self.apply_filters(search)
search.source = {
'query': search.query,
'size': len(item_ids) if size is None else size,
}
if aggregations is not None:
search.source['aggs'] = aggregations
req = ParsedRequest()
req.args = {'source': json.dumps(search.source)}
return self.internal_get(req, None)
except Exception as exc:
logger.error('Error in get_items for query: {}'.format(
json.dumps(search.source)),
exc,
exc_info=True)
def assert_products_query(user_id, args=None, products=None):
with app.test_request_context():
session['user'] = user_id
search = SearchQuery()
if args is None:
service.prefill_search_query(search)
else:
req = ParsedRequest()
req.args = args
service.prefill_search_query(search, req)
service.apply_products_filter(search)
for product in products:
if product.get('query'):
assert {
'query_string': {
'query': product['query'],
'default_operator': 'AND',
'lenient': True
}
} in search.query['bool']['should']
sd_product_ids = [
product['sd_product_id'] for product in products
if product.get('sd_product_id')
]
if len(sd_product_ids):
assert {
'terms': {
'products.code': sd_product_ids
}
} in search.query['bool']['should']
def test_apply_section_filter(client, app):
with app.test_request_context():
session['user'] = ADMIN_USER_ID
search = SearchQuery()
service.section = 'wire'
service.prefill_search_query(search)
service.apply_section_filter(search)
assert {
'query_string': {
'query': SECTION_FILTERS[0]['query'],
'default_operator': 'AND',
'lenient': True
}
} in search.query['bool']['must']
service.section = 'agenda'
service.prefill_search_query(search)
service.apply_section_filter(search)
assert {
'query_string': {
'query': SECTION_FILTERS[1]['query'],
'default_operator': 'AND',
'lenient': True
}
} in search.query['bool']['must']
def _set_search_query(user_id):
with app.test_request_context():
session['user'] = user_id
search = SearchQuery()
service.prefill_search_user(search)
service.prefill_search_company(search)
service.apply_company_filter(search)
return search.query
def test_prefill_search_items(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_query(search)
assert {'term': {'_type': 'items'}} in search.query['bool']['must']
assert {'term': {'type': 'composite'}} in search.query['bool']['must_not']
assert {'constant_score': {
'filter': {'exists': {'field': 'nextversion'}}
}} in search.query['bool']['must_not']
search = SearchQuery()
req = ParsedRequest()
req.args = {'ignore_latest': True}
service.prefill_search_query(search, req)
assert {'constant_score': {
'filter': {'exists': {'field': 'nextversion'}}
}} not in search.query['bool']['must_not']
def test_prefill_search_products__admin_products(client, app):
with app.test_request_context():
session['user'] = ADMIN_USER_ID
search = SearchQuery()
service.prefill_search_query(search)
assert search.products == []
search = SearchQuery()
req = ParsedRequest()
req.args = {'navigation': [
NAV_1, # PROD_1: enabled for Admin/Company
NAV_3, # PROD_2: not enabled for Admin/Company
NAV_4, # PROD_3: is_enabled=False
NAV_5, # PROD_4: product_type=agenda
]}
service.prefill_search_query(search, req)
assert len(search.products) == 2
assert search.products[0]['_id'] in [PROD_1, PROD_2]
assert search.products[1]['_id'] in [PROD_1, PROD_2]
def test_apply_request_filter__query_string(client, app):
with app.test_request_context():
search = SearchQuery()
search.args = {'q': 'Sport AND Tennis'}
service.apply_request_filter(search)
assert {
'query_string': {
'query': 'Sport AND Tennis',
'default_operator': 'AND',
'lenient': True
}
} in search.query['bool']['must']
search.args = {'q': 'Sport AND Tennis', 'default_operator': 'OR'}
service.apply_request_filter(search)
assert {
'query_string': {
'query': 'Sport AND Tennis',
'default_operator': 'OR',
'lenient': True
}
} in search.query['bool']['must']
def test_prefill_search_products__requested_products(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_query(search)
assert search.requested_products == []
search = SearchQuery()
req = ParsedRequest()
req.args = {'requested_products': '{},{},{}'.format(PROD_1, PROD_2, PROD_3)}
service.prefill_search_query(search, req)
assert search.requested_products == [str(PROD_1), str(PROD_2), str(PROD_3)]
search = SearchQuery()
req = ParsedRequest()
req.args = {'requested_products': [str(PROD_1), str(PROD_2), str(PROD_3)]}
service.prefill_search_query(search, req)
assert search.requested_products == [str(PROD_1), str(PROD_2), str(PROD_3)]
search = SearchQuery()
req = ParsedRequest()
req.args = {'requested_products': {'test': PROD_3}}
with raises(BadParameterValueError):
service.prefill_search_query(search, req)
def test_prefill_search_products__public_products(client, app):
with app.test_request_context():
session['user'] = PUBLIC_USER_ID
search = SearchQuery()
service.prefill_search_query(search)
assert len(search.products) == 2
assert search.products[0]['_id'] in [PROD_1, PROD_2]
assert search.products[1]['_id'] in [PROD_1, PROD_2]
search = SearchQuery()
req = ParsedRequest()
req.args = {'navigation': [
NAV_1, # PROD_1: enabled for Public/Company
NAV_2, # PROD_1: enabled for Public/Company
NAV_3, # PROD_2: enabled for Public/Company
NAV_4, # PROD_3: is_enabled=False
NAV_5, # PROD_4: product_type=agenda
NAV_6, # PROD_4: disabled for Public/Company
]}
service.prefill_search_query(search)
assert len(search.products) == 2
assert search.products[0]['_id'] in [PROD_1, PROD_2]
assert search.products[1]['_id'] in [PROD_1, PROD_2]
def test_prefill_search_page(client, app):
with app.test_request_context():
search = SearchQuery()
service.prefill_search_query(search)
assert search.args == {
'sort': service.default_sort,
'size': service.default_page_size,
'from': 0
}
search = SearchQuery()
req = ParsedRequest()
req.args = {
'sort': [{'versioncreated': 'asc'}],
'size': '50',
'from': '50',
}
service.prefill_search_query(search, req)
assert search.args == {
'sort': [{'versioncreated': 'asc'}],
'size': 50,
'from': 50
}
def test_prefill_search_company(client, app):
with app.test_request_context():
session['user'] = None
search = SearchQuery()
service.prefill_search_query(search)
assert search.user is None
assert search.company is None
session['user'] = ADMIN_USER_ID
search = SearchQuery()
service.prefill_search_query(search)
assert search.company.get('_id') == COMPANY_1
session['user'] = PUBLIC_USER_ID
search = SearchQuery()
service.prefill_search_query(search)
assert search.company.get('_id') == COMPANY_2
session['user'] = ADMIN_USER_ID
search = SearchQuery()
req = ParsedRequest()
req.args = {'user': TEST_USER_ID}
service.prefill_search_query(search, req)
assert search.company.get('_id') == COMPANY_3
def get_product_items(self, product_id, size):
search = SearchQuery()
self.prefill_search_args(search)
self.prefill_search_items(search)
search.args['size'] = size
product = get_resource_service('products').find_one(req=None,
_id=product_id)
if not product:
return
search.query['bool']['must'].append({
"bool": {
"should": [{
"range": {
"embargoed": {
"lt": "now"
}
}
}, {
"bool": {
"must_not": {
"exists": {
"field": "embargoed"
}
}
}
}]
}
})
get_resource_service('section_filters').apply_section_filter(
search.query, product.get('product_type'))
search.query['bool']['should'] = []
if product.get('sd_product_id'):
search.query['bool']['should'].append(
{'term': {
'products.code': product['sd_product_id']
}})
if product.get('query'):
search.query['bool']['should'].append(
query_string(product['query']))
search.query['bool']['minimum_should_match'] = 1
self.gen_source_from_search(search)
search.source['post_filter'] = {'bool': {'must': []}}
internal_req = self.get_internal_request(search)
return list(self.internal_get(internal_req, None))
def get_matching_topics(self, item_id, topics, users, companies):
""" Returns a list of topic ids matching to the given item_id
:param item_id: item id to be tested against all topics
:param topics: list of topics
:param users: user_id, user dictionary
:param companies: company_id, company dictionary
:return:
"""
query = {
'bool': {
'must_not': [
{
'term': {
'type': 'composite'
}
},
{
'constant_score': {
'filter': {
'exists': {
'field': 'nextversion'
}
}
}
},
],
'must': [{
'term': {
'_id': item_id
}
}],
'should': []
}
}
aggs = {'topics': {'filters': {'filters': {}}}}
queried_topics = []
# get all section filters
section_filters = get_resource_service(
'section_filters').get_section_filters_dict()
for topic in topics:
search = SearchQuery()
user = users.get(str(topic['user']))
if not user:
continue
search.user = user
search.is_admin = is_admin(user)
search.company = companies.get(str(user.get('company', '')))
search.query = deepcopy(query)
search.query['bool']['must'] = [{'term': {'_id': item_id}}]
search.section = topic.get('topic_type')
self.prefill_search_products(search)
topic_filter = {'bool': {'must': []}}
if topic.get('query'):
topic_filter['bool']['must'].append(
query_string(topic['query']))
if topic.get('created'):
topic_filter['bool']['must'].append(
self.versioncreated_range(
dict(created_from=topic['created'].get('from'),
created_to=topic['created'].get('to'),
timezone_offset=topic.get('timezone_offset',
'0'))))
if topic.get('filter'):
topic_filter['bool']['must'] += self._filter_terms(
topic['filter'])
# for now even if there's no active company matching for the user
# continuing with the search
try:
self.validate_request(search)
self.apply_section_filter(search, section_filters)
self.apply_company_filter(search)
self.apply_time_limit_filter(search)
self.apply_products_filter(search)
except Forbidden:
logger.info(
'Notification for user:{} and topic:{} is skipped'.format(
user.get('_id'), topic.get('_id')))
continue
aggs['topics']['filters']['filters'][str(
topic['_id'])] = topic_filter
queried_topics.append(topic)
source = {'query': query}
source['aggs'] = aggs
source['size'] = 0
req = ParsedRequest()
req.args = {'source': json.dumps(source)}
topic_matches = []
try:
search_results = self.internal_get(req, None)
for topic in queried_topics:
if search_results.hits['aggregations']['topics']['buckets'][
str(topic['_id'])]['doc_count'] > 0:
topic_matches.append(topic['_id'])
except Exception as exc:
logger.error('Error in get_matching_topics for query: {}'.format(
json.dumps(source)),
exc,
exc_info=True)
return topic_matches
def test_apply_request_filter__versioncreated(client, app):
with app.test_request_context():
app.config['FILTER_BY_POST_FILTER'] = False
search = SearchQuery()
search.args = {'created_from': '2020-03-27'}
service.apply_request_filter(search)
assert {
'range': {
'versioncreated': {
'gte': get_local_date('2020-03-27', '00:00:00', 0)
}
}
} in search.query['bool']['must']
search = SearchQuery()
search.args = {'created_to': '2020-03-27'}
service.apply_request_filter(search)
assert {
'range': {
'versioncreated': {
'lte': get_local_date('2020-03-27', '23:59:59', 0)
}
}
} in search.query['bool']['must']
search = SearchQuery()
search.args = {
'created_from': '2020-03-27',
'created_from_time': '01:12:45',
'created_to': '2020-03-27'
}
service.apply_request_filter(search)
assert {
'range': {
'versioncreated': {
'gte': get_local_date('2020-03-27', '01:12:45', 0),
'lte': get_local_date('2020-03-27', '23:59:59', 0)
}
}
} in search.query['bool']['must']
app.config['FILTER_BY_POST_FILTER'] = True
search = SearchQuery()
search.args = {'created_from': '2020-03-27'}
service.apply_request_filter(search)
assert {
'range': {
'versioncreated': {
'gte': get_local_date('2020-03-27', '00:00:00', 0)
}
}
} in search.source['post_filter']['bool']['must']
search = SearchQuery()
search.args = {'created_to': '2020-03-27'}
service.apply_request_filter(search)
assert {
'range': {
'versioncreated': {
'lte': get_local_date('2020-03-27', '23:59:59', 0)
}
}
} in search.source['post_filter']['bool']['must']
search = SearchQuery()
search.args = {
'created_from': '2020-03-27',
'created_from_time': '01:12:45',
'created_to': '2020-03-27'
}
service.apply_request_filter(search)
assert {
'range': {
'versioncreated': {
'gte': get_local_date('2020-03-27', '01:12:45', 0),
'lte': get_local_date('2020-03-27', '23:59:59', 0)
}
}
} in search.source['post_filter']['bool']['must']
def test_apply_request_filter__filters(client, app):
with app.test_request_context():
app.config['FILTER_BY_POST_FILTER'] = False
app.config['FILTER_AGGREGATIONS'] = True
search = SearchQuery()
search.args = {'filter': json.dumps({'service': ['a']})}
service.apply_request_filter(search)
assert {
'terms': {
'service.name': ['a']
}
} in search.query['bool']['must']
search = SearchQuery()
search.args = {'filter': {'service': ['a']}}
service.apply_request_filter(search)
assert {
'terms': {
'service.name': ['a']
}
} in search.query['bool']['must']
with raises(BadParameterValueError):
search.args = {'filter': ['test']}
service.apply_request_filter(search)
app.config['FILTER_BY_POST_FILTER'] = False
app.config['FILTER_AGGREGATIONS'] = False
search = SearchQuery()
search.args = {'filter': {'term': {'service': 'a'}}}
service.apply_request_filter(search)
assert {'term': {'service': 'a'}} in search.query['bool']['must']
app.config['FILTER_BY_POST_FILTER'] = True
app.config['FILTER_AGGREGATIONS'] = True
search = SearchQuery()
search.args = {'filter': {'service': ['a']}}
service.apply_request_filter(search)
assert {
'terms': {
'service.name': ['a']
}
} in search.source['post_filter']['bool']['must']
app.config['FILTER_BY_POST_FILTER'] = True
app.config['FILTER_AGGREGATIONS'] = False
search = SearchQuery()
search.args = {'filter': {'term': {'service': 'a'}}}
service.apply_request_filter(search)
assert {
'term': {
'service': 'a'
}
} in search.source['post_filter']['bool']['must']