def _get_combined_view_data(self, items, request): """Get list of event and planning for the combined view :param items: :param request: object representing the HTTP request """ ids = set() for item in items: # don't want related planing items _id = item.get('event_item') or item.get('_id') ids.add(_id) filters = self._get_date_filters(request) page = request.page or 1 page_size = self._get_page_size(request) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': { 'bool': { 'must': [{'terms': {'_id': list(ids)}}], } }, 'filter': filters, 'sort': self._get_sort(), 'size': self._get_page_size(request), 'from': (page - 1) * page_size }) req.page = request.page or 1 req.max_results = self._get_page_size(request) return get_resource_service('planning_search').get(req=req, lookup=None)
def _get_events_and_planning(self, request, query, search_filter): """Get list of event and planning based on the search criteria :param request: object representing the HTTP request """ # params = request.args or MultiDict() # query = construct_combined_search_query(params) page = request.page or 1 max_results = self._get_page_size(request, search_filter) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': query['query'], 'sort': query['sort'] if query.get('sort') else self._get_sort(), 'size': int((5 * max_results) * math.ceil(page / 3)), }) req.args['projections'] = json.dumps(['_id', 'type', 'event_item']) req.page = page req.max_results = max_results req.exec_on_fetched_resource = False # don't call on_fetched_resource return get_resource_service('planning_search').get(req=req, lookup=None)
def forwards(self, mongodb_collection, mongodb_database): archive_service = get_resource_service('archive') published_service = get_resource_service(self.resource) templates_service = get_resource_service('content_templates') template = templates_service.find_one(req=None, template_name='article') if not template: return req = ParsedRequest() req.max_results = 50 for page in range(1, 200): req.page = page items = list(published_service.get(req=req, lookup=None)) if not items: break for item in items: if ObjectId(item.get('template')) != template.get('_id'): extra = item.get('extra') if extra is not None: extra.pop('compliantlifetime', None) published_service.system_update( ObjectId(item['_id']), {'extra': extra}, item) archive_item = archive_service.find_one( req=None, _id=item.get('item_id')) if archive_item: archive_service.system_update( archive_item.get('_id'), {'extra': extra}, archive_item)
def forwards(self, mongodb_collection, mongodb_database): archive_service = get_resource_service('archive') published_service = get_resource_service(self.resource) req = ParsedRequest() req.max_results = 50 for page in range(1, 200): # 10k limit req.page = page items = list(published_service.get(req=req, lookup=None)) if not items: break for item in items: published_date = item.get('firstpublished') if published_date is not None: compliant_lifetime = add_years(published_date, 1) extra = item.get('extra', {}) extra['compliantlifetime'] = compliant_lifetime try: published_service.system_update( ObjectId(item['_id']), {'extra': extra}, item) except SuperdeskApiError: continue archive_item = archive_service.find_one( req=None, _id=item['item_id']) if archive_item: archive_service.system_update(archive_item['_id'], {'extra': extra}, archive_item)
def test_pagination(data_layer): resource = 'instruments' sub_resource_lookup = None req = ParsedRequest() req.max_results = 1 req.page = 2 results = data_layer.find(resource, req, sub_resource_lookup) assert len(results) == 1
def _get_max_date_from_publish_queue(self): """ Get the max _updated date from legal_publish_queue collection :return datetime: _updated time """ legal_publish_queue_service = get_resource_service(LEGAL_PUBLISH_QUEUE_NAME) req = ParsedRequest() req.sort = '[("%s", 1)]' % config.LAST_UPDATED req.max_results = 1 req.page = 1 queue_item = list(legal_publish_queue_service.get(req=req, lookup={})) return queue_item[0][config.LAST_UPDATED] if queue_item else None
def _get_max_date_from_publish_queue(self): """ Get the max _updated date from legal_publish_queue collection :return datetime: _updated time """ legal_publish_queue_service = get_resource_service( LEGAL_PUBLISH_QUEUE_NAME) req = ParsedRequest() req.sort = '[("%s", -1)]' % config.LAST_UPDATED req.max_results = 1 req.page = 1 queue_item = list(legal_publish_queue_service.get(req=req, lookup={})) return queue_item[0][config.LAST_UPDATED] if queue_item else None
def search_repos(self, repo, args, page=1, page_size=None, projections=None): req = ParsedRequest() req.args = MultiDict() req.args['repo'] = repo req.args.update(args) if projections is not None: req.args['projections'] = json.dumps(projections) req.page = page req.max_results = page_size or self.default_page_size return self.get(req=req, lookup=None)
def get_series(self, query, sort, max_results): page = 1 while True: # Get the results from mongo req = ParsedRequest() req.sort = sort req.where = json.dumps(query) req.max_results = max_results req.page = page results = self.get_from_mongo(req=req, lookup=None) docs = list(results) if not docs: break page += 1 # Yield the results for iteration by the callee for doc in docs: yield doc
def _search_events(self, request, params, query, search_filter): page = request.page or 1 page_size = self._get_page_size(request, search_filter) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': query['query'], 'sort': query['sort'] if query.get('sort') else { 'dates.start': { 'order': 'asc' } }, 'size': page_size, 'from': (page - 1) * page_size }) req.args['repos'] = 'events' req.page = page req.max_results = page_size if params.get('projections'): req.args['projections'] = params['projections'] return get_resource_service('planning_search').get(req=req, lookup=None)
def _search_planning(self, request, params, query, search_filter): # params = request.args or MultiDict() # query = construct_planning_search_query(params) page = request.page or 1 page_size = self._get_page_size(request, search_filter) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': query['query'], 'sort': query['sort'] if query.get('sort') else self._get_sort(), 'size': page_size, 'from': (page - 1) * page_size }) req.args['repos'] = 'planning' req.page = page req.max_results = page_size if params.get('projections'): req.args['projections'] = params['projections'] return get_resource_service('planning_search').get(req=req, lookup=None)
def _get_combined_view_data(self, items, request, params, search_filter): """Get list of event and planning for the combined view :param items: :param request: object representing the HTTP request """ query = construct_combined_view_data_query(params, search_filter, items) page = request.page or 1 page_size = self._get_page_size(request, search_filter) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': query['query'], 'sort': self._get_sort(), 'size': page_size, 'from': (page - 1) * page_size }) req.page = request.page or 1 req.max_results = page_size if params.get('projections'): req.args['projections'] = params['projections'] return get_resource_service('planning_search').get(req=req, lookup=None)