def get_agenda_query(query, events_only=False):
if events_only:
return query_string(query)
else:
return {
'or': [
query_string(query),
nested_query(
'planning_items',
planning_items_query_string(query),
name='query'
)
]
}
def planning_items_query_string(query, fields=None):
plan_query_string = query_string(query)
if fields:
plan_query_string['query_string']['fields'] = fields
else:
plan_query_string['query_string']['fields'] = ['planning_items.*']
return plan_query_string
def apply_request_filter(self, search):
""" Generate the filters from request args
:param newsroom.search.SearchQuery search: The search query instance
"""
if search.args.get('q'):
search.query['bool']['must'].append(
query_string(search.args['q'],
search.args.get('default_operator') or 'AND'))
def get_product_items(self, product_id, size):
search = SearchQuery()
self.prefill_search_args(search)
self.prefill_search_items(search)
search.args['size'] = size
product = get_resource_service('products').find_one(req=None,
_id=product_id)
if not product:
return
search.query['bool']['must'].append({
"bool": {
"should": [{
"range": {
"embargoed": {
"lt": "now"
}
}
}, {
"bool": {
"must_not": {
"exists": {
"field": "embargoed"
}
}
}
}]
}
})
get_resource_service('section_filters').apply_section_filter(
search.query, product.get('product_type'))
search.query['bool']['should'] = []
if product.get('sd_product_id'):
search.query['bool']['should'].append(
{'term': {
'products.code': product['sd_product_id']
}})
if product.get('query'):
search.query['bool']['should'].append(
query_string(product['query']))
search.query['bool']['minimum_should_match'] = 1
self.gen_source_from_search(search)
search.source['post_filter'] = {'bool': {'must': []}}
internal_req = self.get_internal_request(search)
return list(self.internal_get(internal_req, None))
def get_navigation_story_count(self, navigations, section, company, user):
"""Get story count by navigation"""
search = SearchQuery()
self.prefill_search_args(search)
self.prefill_search_items(search)
search.section = section
search.user = user
search.company = company
self.apply_section_filter(search)
aggs = {'navigations': {'filters': {'filters': {}}}}
for navigation in navigations:
navigation_id = navigation.get('_id')
products = get_products_by_navigation(navigation_id) or []
navigation_filter = {
'bool': {
'should': [],
'minimum_should_match': 1
}
}
for product in products:
if product.get('query'):
navigation_filter['bool']['should'].append(
query_string(product.get('query')))
if navigation_filter['bool']['should']:
aggs['navigations']['filters']['filters'][str(
navigation_id)] = navigation_filter
source = {'query': search.query, 'aggs': aggs, 'size': 0}
req = ParsedRequest()
req.args = {'source': json.dumps(source)}
try:
results = self.internal_get(req, None)
buckets = results.hits['aggregations']['navigations']['buckets']
for navigation in navigations:
navigation_id = navigation.get('_id')
doc_count = buckets.get(str(navigation_id),
{}).get('doc_count', 0)
if doc_count > 0:
navigation['story_count'] = doc_count
except Exception as exc:
logger.error(
'Error in get_navigation_story_count for query: {}'.format(
json.dumps(source)),
exc,
exc_info=True)
def apply_request_filter(self, search):
""" Generate the request filters
:param newsroom.search.SearchQuery search: The search query instance
"""
if search.args.get('q'):
test_query = {'or': []}
try:
q = json.loads(search.args.get('q'))
if isinstance(q, dict):
# used for product testing
if q.get('query'):
test_query['or'].append(query_string(q.get('query')))
if q.get('planning_item_query'):
test_query['or'].append(
nested_query(
'planning_items',
planning_items_query_string(q.get('planning_item_query')),
name='product_test'
)
)
if test_query['or']:
search.query['bool']['must'].append(test_query)
except Exception:
pass
if not test_query.get('or'):
search.query['bool']['must'].append(
get_agenda_query(
search.args['q'],
search.is_events_only
)
)
if search.args.get('id'):
search.query['bool']['must'].append(
{'term': {'_id': search.args['id']}}
)
if search.args.get('bookmarks'):
set_saved_items_query(
search.query,
search.args['bookmarks']
)
if search.args.get('date_from') or search.args.get('date_to'):
_set_event_date_range(search)
def apply_section_filter(self, query, product_type, filters=None):
"""Get the list of base products for product type
:param query: Dict of elasticsearch query
:param product_type: Type of product
:param filters: filters for each section
"""
if not filters:
section_filters = self.get_section_filters(product_type)
else:
section_filters = filters.get(product_type)
if not section_filters:
return
for f in section_filters:
if f.get('query'):
query['bool']['must'].append(query_string(f.get('query')))
def apply_products_filter(self, search):
""" Generate the product filters
:param newsroom.wire.service.SearchQuery search: the search query instance
"""
monitoring_list = []
if search.req:
if len(search.navigation_ids) > 0:
monitoring_list.append(
get_resource_service('monitoring').find_one(
req=None, _id=search.navigation_ids[0]))
else:
abort(403, gettext('No monitoring profile requested.'))
else:
monitoring_list = list(query_resource('monitoring'))
if len(monitoring_list) < 1:
return
for mlist in monitoring_list:
search.query['bool']['should'].append(query_string(mlist['query']))
if search.navigation_ids and len(monitoring_list[0].get('keywords') or
[]) > 0 and search.source is not None:
search.source['highlight'] = {'fields': {}}
fields = ['body_html']
for field in fields:
search.source['highlight']['fields'][field] = {
"number_of_fragments": 0,
"highlight_query": {
"query_string": {
"query": ' '.join(monitoring_list[0]['keywords']),
"default_operator": "AND",
"lenient": False
}
}
}
search.source['highlight']['pre_tags'] = [
"<span class='es-highlight'>"
]
search.source['highlight']['post_tags'] = ["</span>"]
search.source['highlight']['require_field_match'] = False
def apply_request_filter(self, search):
""" Generate the filters from request args
:param newsroom.search.SearchQuery search: The search query instance
"""
super().apply_request_filter(search)
if search.args.get('bookmarks'):
set_bookmarks_query(search.query, search.args['bookmarks'])
if search.args.get('newsOnly') and not (
search.args.get('navigation')
or search.args.get('product_type')):
news_only_filter = get_setting('news_only_filter')
if news_only_filter:
search.query['bool']['must_not'].append(
query_string(news_only_filter))
elif app.config.get('NEWS_ONLY_FILTERS'):
for f in app.config.get('NEWS_ONLY_FILTERS', []):
search.query['bool']['must_not'].append(f)
def apply_product_filter(self, search, product):
""" Generate the filter for a single product
:param newsroom.search.SearchQuery search: The search query instance
:param dict product: The product to filter
:return:
"""
if search.args.get('requested_products') and product['_id'] not in search.args['requested_products']:
return
if product.get('query'):
search.query['bool']['should'].append(
query_string(product['query'])
)
if product.get('planning_item_query') and not search.is_events_only:
search.planning_items_should.append(
planning_items_query_string(
product.get('planning_item_query')
)
)
def get_product_item_report(self, product, section_filters=None):
query = items_query()
if not product:
return
query['bool']['should'] = []
get_resource_service('section_filters').apply_section_filter(
query, product.get('product_type'), section_filters)
if product.get('sd_product_id'):
query['bool']['should'].append(
{'term': {
'products.code': product['sd_product_id']
}})
if product.get('query'):
query['bool']['should'].append(query_string(product['query']))
query['bool']['minimum_should_match'] = 1
query['bool']['must_not'].append({'term': {'pubstatus': 'canceled'}})
now = datetime.utcnow()
source = {'query': query}
source['size'] = 0
source['aggs'] = {
"today": {
"date_range": {
"field": "versioncreated",
"ranges": [{
"from": now.strftime('%Y-%m-%d')
}]
}
},
"last_24_hours": {
"date_range": {
"field": "versioncreated",
"ranges": [{
"from": "now-1d/d"
}]
}
},
"this_week": {
"date_range": {
"field":
"versioncreated",
"ranges": [{
"from":
(now -
timedelta(days=now.weekday())).strftime('%Y-%m-%d')
}]
}
},
"last_7_days": {
"date_range": {
"field":
"versioncreated",
"ranges": [{
"from": (now - timedelta(days=7)).strftime('%Y-%m-%d')
}]
}
},
"this_month": {
"date_range": {
"field": "versioncreated",
"ranges": [{
"from": (now.replace(day=1)).strftime('%Y-%m-%d')
}]
}
},
"previous_month": {
"date_range": {
"field":
"versioncreated",
"ranges": [{
"from":
(((now.replace(day=1)) - timedelta(days=1)).replace(
day=1)).strftime('%Y-%m-%d'),
"to": (now.replace(day=1)).strftime('%Y-%m-%d'),
}]
}
},
"last_6_months": {
"date_range": {
"field":
"versioncreated",
"ranges": [{
"from":
(now - timedelta(days=180)).strftime('%Y-%m-%d')
}]
}
},
}
internal_req = ParsedRequest()
internal_req.args = {'source': json.dumps(source)}
return self.internal_get(internal_req, None)
def get_matching_topics(self, item_id, topics, users, companies):
""" Returns a list of topic ids matching to the given item_id
:param item_id: item id to be tested against all topics
:param topics: list of topics
:param users: user_id, user dictionary
:param companies: company_id, company dictionary
:return:
"""
query = {
'bool': {
'must_not': [
{
'term': {
'type': 'composite'
}
},
{
'constant_score': {
'filter': {
'exists': {
'field': 'nextversion'
}
}
}
},
],
'must': [{
'term': {
'_id': item_id
}
}],
'should': []
}
}
aggs = {'topics': {'filters': {'filters': {}}}}
queried_topics = []
# get all section filters
section_filters = get_resource_service(
'section_filters').get_section_filters_dict()
for topic in topics:
search = SearchQuery()
user = users.get(str(topic['user']))
if not user:
continue
search.user = user
search.is_admin = is_admin(user)
search.company = companies.get(str(user.get('company', '')))
search.query = deepcopy(query)
search.query['bool']['must'] = [{'term': {'_id': item_id}}]
search.section = topic.get('topic_type')
self.prefill_search_products(search)
topic_filter = {'bool': {'must': []}}
if topic.get('query'):
topic_filter['bool']['must'].append(
query_string(topic['query']))
if topic.get('created'):
topic_filter['bool']['must'].append(
self.versioncreated_range(
dict(created_from=topic['created'].get('from'),
created_to=topic['created'].get('to'),
timezone_offset=topic.get('timezone_offset',
'0'))))
if topic.get('filter'):
topic_filter['bool']['must'] += self._filter_terms(
topic['filter'])
# for now even if there's no active company matching for the user
# continuing with the search
try:
self.validate_request(search)
self.apply_section_filter(search, section_filters)
self.apply_company_filter(search)
self.apply_time_limit_filter(search)
self.apply_products_filter(search)
except Forbidden:
logger.info(
'Notification for user:{} and topic:{} is skipped'.format(
user.get('_id'), topic.get('_id')))
continue
aggs['topics']['filters']['filters'][str(
topic['_id'])] = topic_filter
queried_topics.append(topic)
source = {'query': query}
source['aggs'] = aggs
source['size'] = 0
req = ParsedRequest()
req.args = {'source': json.dumps(source)}
topic_matches = []
try:
search_results = self.internal_get(req, None)
for topic in queried_topics:
if search_results.hits['aggregations']['topics']['buckets'][
str(topic['_id'])]['doc_count'] > 0:
topic_matches.append(topic['_id'])
except Exception as exc:
logger.error('Error in get_matching_topics for query: {}'.format(
json.dumps(source)),
exc,
exc_info=True)
return topic_matches
def featured(self, req, lookup, featured):
"""Return featured items.
:param ParsedRequest req: The parsed in request instance from the endpoint
:param dict lookup: The parsed in lookup dictionary from the endpoint
:param dict featured: list featured items
"""
user = get_user()
company = get_user_company(user)
if is_events_only_access(user, company):
abort(403)
if not featured or not featured.get('items'):
return ListCursor([])
query = _agenda_query()
get_resource_service('section_filters').apply_section_filter(query, self.section)
planning_items_query = nested_query(
'planning_items',
{
'bool': {'must': [{'terms': {'planning_items.guid': featured['items']}}]}
},
name='featured'
)
if req.args.get('q'):
query['bool']['must'].append(query_string(req.args['q']))
planning_items_query['nested']['query']['bool']['must'].append(planning_items_query_string(req.args['q']))
query['bool']['must'].append(planning_items_query)
source = {'query': query}
set_post_filter(source, req)
source['size'] = len(featured['items'])
source['from'] = req.args.get('from', 0, type=int)
if not source['from']:
source['aggs'] = aggregations
if company and not is_admin(user) and company.get('events_only', False):
# no adhoc planning items and remove planning items and coverages fields
query['bool']['must'].append({'exists': {'field': 'event'}})
_remove_fields(source, PLANNING_ITEMS_FIELDS)
internal_req = ParsedRequest()
internal_req.args = {'source': json.dumps(source)}
cursor = self.internal_get(internal_req, lookup)
docs_by_id = {}
for doc in cursor.docs:
for p in (doc.get('planning_items') or []):
docs_by_id[p.get('guid')] = doc
# make the items display on the featured day,
# it's used in ui instead of dates.start and dates.end
doc.update({
'_display_from': featured['display_from'],
'_display_to': featured['display_to'],
})
docs = []
agenda_ids = set()
for _id in featured['items']:
if docs_by_id.get(_id) and docs_by_id.get(_id).get('_id') not in agenda_ids:
docs.append(docs_by_id.get(_id))
agenda_ids.add(docs_by_id.get(_id).get('_id'))
cursor.docs = docs
return cursor
