def get_agenda_query(query, events_only=False): if events_only: return query_string(query) else: return { 'or': [ query_string(query), nested_query( 'planning_items', planning_items_query_string(query), name='query' ) ] }
def planning_items_query_string(query, fields=None): plan_query_string = query_string(query) if fields: plan_query_string['query_string']['fields'] = fields else: plan_query_string['query_string']['fields'] = ['planning_items.*'] return plan_query_string
def apply_request_filter(self, search): """ Generate the filters from request args :param newsroom.search.SearchQuery search: The search query instance """ if search.args.get('q'): search.query['bool']['must'].append( query_string(search.args['q'], search.args.get('default_operator') or 'AND'))
def get_product_items(self, product_id, size): search = SearchQuery() self.prefill_search_args(search) self.prefill_search_items(search) search.args['size'] = size product = get_resource_service('products').find_one(req=None, _id=product_id) if not product: return search.query['bool']['must'].append({ "bool": { "should": [{ "range": { "embargoed": { "lt": "now" } } }, { "bool": { "must_not": { "exists": { "field": "embargoed" } } } }] } }) get_resource_service('section_filters').apply_section_filter( search.query, product.get('product_type')) search.query['bool']['should'] = [] if product.get('sd_product_id'): search.query['bool']['should'].append( {'term': { 'products.code': product['sd_product_id'] }}) if product.get('query'): search.query['bool']['should'].append( query_string(product['query'])) search.query['bool']['minimum_should_match'] = 1 self.gen_source_from_search(search) search.source['post_filter'] = {'bool': {'must': []}} internal_req = self.get_internal_request(search) return list(self.internal_get(internal_req, None))
def get_navigation_story_count(self, navigations, section, company, user): """Get story count by navigation""" search = SearchQuery() self.prefill_search_args(search) self.prefill_search_items(search) search.section = section search.user = user search.company = company self.apply_section_filter(search) aggs = {'navigations': {'filters': {'filters': {}}}} for navigation in navigations: navigation_id = navigation.get('_id') products = get_products_by_navigation(navigation_id) or [] navigation_filter = { 'bool': { 'should': [], 'minimum_should_match': 1 } } for product in products: if product.get('query'): navigation_filter['bool']['should'].append( query_string(product.get('query'))) if navigation_filter['bool']['should']: aggs['navigations']['filters']['filters'][str( navigation_id)] = navigation_filter source = {'query': search.query, 'aggs': aggs, 'size': 0} req = ParsedRequest() req.args = {'source': json.dumps(source)} try: results = self.internal_get(req, None) buckets = results.hits['aggregations']['navigations']['buckets'] for navigation in navigations: navigation_id = navigation.get('_id') doc_count = buckets.get(str(navigation_id), {}).get('doc_count', 0) if doc_count > 0: navigation['story_count'] = doc_count except Exception as exc: logger.error( 'Error in get_navigation_story_count for query: {}'.format( json.dumps(source)), exc, exc_info=True)
def apply_request_filter(self, search): """ Generate the request filters :param newsroom.search.SearchQuery search: The search query instance """ if search.args.get('q'): test_query = {'or': []} try: q = json.loads(search.args.get('q')) if isinstance(q, dict): # used for product testing if q.get('query'): test_query['or'].append(query_string(q.get('query'))) if q.get('planning_item_query'): test_query['or'].append( nested_query( 'planning_items', planning_items_query_string(q.get('planning_item_query')), name='product_test' ) ) if test_query['or']: search.query['bool']['must'].append(test_query) except Exception: pass if not test_query.get('or'): search.query['bool']['must'].append( get_agenda_query( search.args['q'], search.is_events_only ) ) if search.args.get('id'): search.query['bool']['must'].append( {'term': {'_id': search.args['id']}} ) if search.args.get('bookmarks'): set_saved_items_query( search.query, search.args['bookmarks'] ) if search.args.get('date_from') or search.args.get('date_to'): _set_event_date_range(search)
def apply_section_filter(self, query, product_type, filters=None): """Get the list of base products for product type :param query: Dict of elasticsearch query :param product_type: Type of product :param filters: filters for each section """ if not filters: section_filters = self.get_section_filters(product_type) else: section_filters = filters.get(product_type) if not section_filters: return for f in section_filters: if f.get('query'): query['bool']['must'].append(query_string(f.get('query')))
def apply_products_filter(self, search): """ Generate the product filters :param newsroom.wire.service.SearchQuery search: the search query instance """ monitoring_list = [] if search.req: if len(search.navigation_ids) > 0: monitoring_list.append( get_resource_service('monitoring').find_one( req=None, _id=search.navigation_ids[0])) else: abort(403, gettext('No monitoring profile requested.')) else: monitoring_list = list(query_resource('monitoring')) if len(monitoring_list) < 1: return for mlist in monitoring_list: search.query['bool']['should'].append(query_string(mlist['query'])) if search.navigation_ids and len(monitoring_list[0].get('keywords') or []) > 0 and search.source is not None: search.source['highlight'] = {'fields': {}} fields = ['body_html'] for field in fields: search.source['highlight']['fields'][field] = { "number_of_fragments": 0, "highlight_query": { "query_string": { "query": ' '.join(monitoring_list[0]['keywords']), "default_operator": "AND", "lenient": False } } } search.source['highlight']['pre_tags'] = [ "<span class='es-highlight'>" ] search.source['highlight']['post_tags'] = ["</span>"] search.source['highlight']['require_field_match'] = False
def apply_request_filter(self, search): """ Generate the filters from request args :param newsroom.search.SearchQuery search: The search query instance """ super().apply_request_filter(search) if search.args.get('bookmarks'): set_bookmarks_query(search.query, search.args['bookmarks']) if search.args.get('newsOnly') and not ( search.args.get('navigation') or search.args.get('product_type')): news_only_filter = get_setting('news_only_filter') if news_only_filter: search.query['bool']['must_not'].append( query_string(news_only_filter)) elif app.config.get('NEWS_ONLY_FILTERS'): for f in app.config.get('NEWS_ONLY_FILTERS', []): search.query['bool']['must_not'].append(f)
def apply_product_filter(self, search, product): """ Generate the filter for a single product :param newsroom.search.SearchQuery search: The search query instance :param dict product: The product to filter :return: """ if search.args.get('requested_products') and product['_id'] not in search.args['requested_products']: return if product.get('query'): search.query['bool']['should'].append( query_string(product['query']) ) if product.get('planning_item_query') and not search.is_events_only: search.planning_items_should.append( planning_items_query_string( product.get('planning_item_query') ) )
def get_product_item_report(self, product, section_filters=None): query = items_query() if not product: return query['bool']['should'] = [] get_resource_service('section_filters').apply_section_filter( query, product.get('product_type'), section_filters) if product.get('sd_product_id'): query['bool']['should'].append( {'term': { 'products.code': product['sd_product_id'] }}) if product.get('query'): query['bool']['should'].append(query_string(product['query'])) query['bool']['minimum_should_match'] = 1 query['bool']['must_not'].append({'term': {'pubstatus': 'canceled'}}) now = datetime.utcnow() source = {'query': query} source['size'] = 0 source['aggs'] = { "today": { "date_range": { "field": "versioncreated", "ranges": [{ "from": now.strftime('%Y-%m-%d') }] } }, "last_24_hours": { "date_range": { "field": "versioncreated", "ranges": [{ "from": "now-1d/d" }] } }, "this_week": { "date_range": { "field": "versioncreated", "ranges": [{ "from": (now - timedelta(days=now.weekday())).strftime('%Y-%m-%d') }] } }, "last_7_days": { "date_range": { "field": "versioncreated", "ranges": [{ "from": (now - timedelta(days=7)).strftime('%Y-%m-%d') }] } }, "this_month": { "date_range": { "field": "versioncreated", "ranges": [{ "from": (now.replace(day=1)).strftime('%Y-%m-%d') }] } }, "previous_month": { "date_range": { "field": "versioncreated", "ranges": [{ "from": (((now.replace(day=1)) - timedelta(days=1)).replace( day=1)).strftime('%Y-%m-%d'), "to": (now.replace(day=1)).strftime('%Y-%m-%d'), }] } }, "last_6_months": { "date_range": { "field": "versioncreated", "ranges": [{ "from": (now - timedelta(days=180)).strftime('%Y-%m-%d') }] } }, } internal_req = ParsedRequest() internal_req.args = {'source': json.dumps(source)} return self.internal_get(internal_req, None)
def get_matching_topics(self, item_id, topics, users, companies): """ Returns a list of topic ids matching to the given item_id :param item_id: item id to be tested against all topics :param topics: list of topics :param users: user_id, user dictionary :param companies: company_id, company dictionary :return: """ query = { 'bool': { 'must_not': [ { 'term': { 'type': 'composite' } }, { 'constant_score': { 'filter': { 'exists': { 'field': 'nextversion' } } } }, ], 'must': [{ 'term': { '_id': item_id } }], 'should': [] } } aggs = {'topics': {'filters': {'filters': {}}}} queried_topics = [] # get all section filters section_filters = get_resource_service( 'section_filters').get_section_filters_dict() for topic in topics: search = SearchQuery() user = users.get(str(topic['user'])) if not user: continue search.user = user search.is_admin = is_admin(user) search.company = companies.get(str(user.get('company', ''))) search.query = deepcopy(query) search.query['bool']['must'] = [{'term': {'_id': item_id}}] search.section = topic.get('topic_type') self.prefill_search_products(search) topic_filter = {'bool': {'must': []}} if topic.get('query'): topic_filter['bool']['must'].append( query_string(topic['query'])) if topic.get('created'): topic_filter['bool']['must'].append( self.versioncreated_range( dict(created_from=topic['created'].get('from'), created_to=topic['created'].get('to'), timezone_offset=topic.get('timezone_offset', '0')))) if topic.get('filter'): topic_filter['bool']['must'] += self._filter_terms( topic['filter']) # for now even if there's no active company matching for the user # continuing with the search try: self.validate_request(search) self.apply_section_filter(search, section_filters) self.apply_company_filter(search) self.apply_time_limit_filter(search) self.apply_products_filter(search) except Forbidden: logger.info( 'Notification for user:{} and topic:{} is skipped'.format( user.get('_id'), topic.get('_id'))) continue aggs['topics']['filters']['filters'][str( topic['_id'])] = topic_filter queried_topics.append(topic) source = {'query': query} source['aggs'] = aggs source['size'] = 0 req = ParsedRequest() req.args = {'source': json.dumps(source)} topic_matches = [] try: search_results = self.internal_get(req, None) for topic in queried_topics: if search_results.hits['aggregations']['topics']['buckets'][ str(topic['_id'])]['doc_count'] > 0: topic_matches.append(topic['_id']) except Exception as exc: logger.error('Error in get_matching_topics for query: {}'.format( json.dumps(source)), exc, exc_info=True) return topic_matches
def featured(self, req, lookup, featured): """Return featured items. :param ParsedRequest req: The parsed in request instance from the endpoint :param dict lookup: The parsed in lookup dictionary from the endpoint :param dict featured: list featured items """ user = get_user() company = get_user_company(user) if is_events_only_access(user, company): abort(403) if not featured or not featured.get('items'): return ListCursor([]) query = _agenda_query() get_resource_service('section_filters').apply_section_filter(query, self.section) planning_items_query = nested_query( 'planning_items', { 'bool': {'must': [{'terms': {'planning_items.guid': featured['items']}}]} }, name='featured' ) if req.args.get('q'): query['bool']['must'].append(query_string(req.args['q'])) planning_items_query['nested']['query']['bool']['must'].append(planning_items_query_string(req.args['q'])) query['bool']['must'].append(planning_items_query) source = {'query': query} set_post_filter(source, req) source['size'] = len(featured['items']) source['from'] = req.args.get('from', 0, type=int) if not source['from']: source['aggs'] = aggregations if company and not is_admin(user) and company.get('events_only', False): # no adhoc planning items and remove planning items and coverages fields query['bool']['must'].append({'exists': {'field': 'event'}}) _remove_fields(source, PLANNING_ITEMS_FIELDS) internal_req = ParsedRequest() internal_req.args = {'source': json.dumps(source)} cursor = self.internal_get(internal_req, lookup) docs_by_id = {} for doc in cursor.docs: for p in (doc.get('planning_items') or []): docs_by_id[p.get('guid')] = doc # make the items display on the featured day, # it's used in ui instead of dates.start and dates.end doc.update({ '_display_from': featured['display_from'], '_display_to': featured['display_to'], }) docs = [] agenda_ids = set() for _id in featured['items']: if docs_by_id.get(_id) and docs_by_id.get(_id).get('_id') not in agenda_ids: docs.append(docs_by_id.get(_id)) agenda_ids.add(docs_by_id.get(_id).get('_id')) cursor.docs = docs return cursor