def test_should_highlight(self): with self.app.app_context(): req = ParsedRequest() req.args = {'es_highlight': 1} self.assertTrue(self.app.data.should_highlight(req)) req.args = {'es_highlight': '0'} self.assertFalse(self.app.data.should_highlight(req))
def test_should_aggregate(self): with self.app.app_context(): self.app.config['ELASTICSEARCH_AUTO_AGGREGATIONS'] = False req = ParsedRequest() req.args = {'aggregations': 1} self.assertTrue(self.app.data.should_aggregate(req)) req.args = {'aggregations': '0'} self.assertFalse(self.app.data.should_aggregate(req))
def get_expired_items(self, page_size): """Get expired item that are not moved to legal :return: """ query = { "query": { "filtered": { "filter": { "and": [ {"range": {"expiry": {"lt": "now"}}}, {"term": {"moved_to_legal": False}}, {"not": {"term": {"state": CONTENT_STATE.SCHEDULED}}}, ] } } } } service = get_resource_service("published") req = ParsedRequest() req.args = {"source": json.dumps(query)} req.sort = '[("publish_sequence_no", 1)]' cursor = service.get(req=req, lookup=None) count = cursor.count() no_of_pages = 0 if count: no_of_pages = len(range(0, count, page_size)) sequence_no = cursor[0]["publish_sequence_no"] logger.info("Number of items to move to legal archive: {}, pages={}".format(count, no_of_pages)) for page in range(0, no_of_pages): logger.info( "Fetching published items " "for page number: {} sequence no: {}".format((page + 1), sequence_no) ) req = ParsedRequest() page_query = deepcopy(query) sequence_filter = {"range": {"publish_sequence_no": {"gte": sequence_no}}} if page == 0: sequence_filter = {"range": {"publish_sequence_no": {"gte": sequence_no}}} else: sequence_filter = {"range": {"publish_sequence_no": {"gt": sequence_no}}} page_query["query"]["filtered"]["filter"]["and"].append(sequence_filter) req.args = {"source": json.dumps(page_query)} req.sort = '[("publish_sequence_no", 1)]' req.max_results = page_size cursor = service.get(req=req, lookup=None) items = list(cursor) if len(items): sequence_no = items[len(items) - 1]["publish_sequence_no"] logger.info( "Fetched No. of Items: {} for page: {} " "For import into legal archive.".format(len(items), (page + 1)) ) yield items
def test_should_project(self): with self.app.app_context(): req = ParsedRequest() req.args = {'projections': json.dumps(["priority", "urgency", "word_count", "slugline", "highlights"])} self.assertTrue(self.app.data.should_project(req)) req.args = {'projections': json.dumps([])} self.assertFalse(self.app.data.should_project(req)) req.args = {} self.assertFalse(self.app.data.should_project(req))
def get_expired_items(self, page_size): """Get expired item that are not moved to legal :return: """ query = { 'query': { 'filtered': { 'filter': { 'and': [ {'range': {'expiry': {'lt': 'now'}}}, {'term': {'moved_to_legal': False}}, {'not': {'term': {'state': CONTENT_STATE.SCHEDULED}}} ] } } } } service = get_resource_service('published') req = ParsedRequest() req.args = {'source': json.dumps(query)} req.sort = '[("publish_sequence_no", 1)]' cursor = service.get(req=req, lookup=None) count = cursor.count() no_of_pages = 0 if count: no_of_pages = len(range(0, count, page_size)) sequence_no = cursor[0]['publish_sequence_no'] logger.info('Number of items to move to legal archive: {}, pages={}'.format(count, no_of_pages)) for page in range(0, no_of_pages): logger.info('Fetching published items ' 'for page number: {} sequence no: {}'. format((page + 1), sequence_no)) req = ParsedRequest() page_query = deepcopy(query) sequence_filter = {'range': {'publish_sequence_no': {'gte': sequence_no}}} if page == 0: sequence_filter = {'range': {'publish_sequence_no': {'gte': sequence_no}}} else: sequence_filter = {'range': {'publish_sequence_no': {'gt': sequence_no}}} page_query['query']['filtered']['filter']['and'].append(sequence_filter) req.args = {'source': json.dumps(page_query)} req.sort = '[("publish_sequence_no", 1)]' req.max_results = page_size cursor = service.get(req=req, lookup=None) items = list(cursor) if len(items): sequence_no = items[len(items) - 1]['publish_sequence_no'] logger.info('Fetched No. of Items: {} for page: {} ' 'For import into legal archive.'.format(len(items), (page + 1))) yield items
def test_resource_aggregates_no_auto(self): with self.app.app_context(): self.app.data.insert('items_with_description', [{'uri': 'foo'}]) self.app.config['ELASTICSEARCH_AUTO_AGGREGATIONS'] = False req = ParsedRequest() req.args = {} response = {} cursor = self.app.data.find('items_with_description', req, {}) cursor.extra(response) self.assertNotIn('_aggregations', response) req.args = {'aggregations': 1} cursor = self.app.data.find('items_with_description', req, {}) cursor.extra(response) self.assertIn('_aggregations', response)
def test_aggs_with_request(self): params = { 'params': {'dates': {'filter': 'yesterday'}}, 'aggs': { 'group': {'field': 'anpa_category.qcode'}, 'subgroup': {'field': 'urgency'} }, 'repo': 'published', 'return_type': 'text/csv' } expected_args = { 'params': params['params'], 'aggs': params['aggs'], 'repo': params['repo'], 'return_type': params['return_type'] } request = ParsedRequest() # Request object with source as a json in string format request.args = { 'params': json.dumps(params['params']), 'repo': params['repo'], 'return_type': params['return_type'], 'aggs': json.dumps(params['aggs']) } args = self.service._get_request_or_lookup(req=request, lookup=None) self.assertEqual(args, expected_args) # Request object with args an ImmutableMultiDict request.args = ImmutableMultiDict({ 'params': params['params'], 'repo': params['repo'], 'return_type': params['return_type'], 'aggs': params['aggs'] }) args = self.service._get_request_or_lookup(req=request, lookup=None) self.assertEqual(args, expected_args) # Request object with source as a normal dict request.args = { 'params': params['params'], 'repo': params['repo'], 'return_type': params['return_type'], 'aggs': params['aggs'] } args = self.service._get_request_or_lookup(req=request, lookup=None) self.assertEqual(args, expected_args)
def _get_broadcast_items(self, ids, include_archived_repo=False): """ Get the broadcast items for the master_id and takes_package_id :param list ids: list of item ids :param include_archived_repo True if archived repo needs to be included in search, default is False :return list: list of broadcast items """ query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': {'term': {'genre.name': BROADCAST_GENRE}}, 'should': [ {'terms': {'broadcast.master_id': ids}}, {'terms': {'broadcast.takes_package_id': ids}} ] } } } } } req = ParsedRequest() repos = 'archive,published' if include_archived_repo: repos = 'archive,published,archived' req.args = {'source': json.dumps(query), 'repo': repos} return get_resource_service('search').get(req=req, lookup=None)
def get_publish_items(item_id, last_version): query = {'query': {'filtered': {'filter': {'and': [ {'term': {'item_id': item_id}}, {'term': {LAST_PUBLISHED_VERSION: last_version}} ]}}}} request = ParsedRequest() request.args = {'source': json.dumps(query), 'aggregations': 0} return self.app.data.find(PUBLISHED, req=request, lookup=None)
def get_elastic_item(self, id): resource = superdesk.get_resource_service(self.resource_name) query = {'query': {'filtered': {'filter': {'term': {'_id': id}}}}} request = ParsedRequest() request.args = {'source': json.dumps(query)} items = resource.get(req=request, lookup=None) return items[0]
def get_last_published_version(self, _id): """Returns the last published entry for the passed item id :param _id: :return: """ try: query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'item_id': _id } }, { 'term': { LAST_PUBLISHED_VERSION: True } }] } } } } } request = ParsedRequest() request.args = {'source': json.dumps(query), 'repo': 'published'} items = list(self.get(req=request, lookup=None)) if items: return items[0] except Exception: return None
def can_publish_take(self, package, sequence): """ Takes can be published only in ascending order. This function check that if there are any unpublished takes before the current take. :param dict package: takes packages :param int sequence: take sequence of the published take :return: True if takes are published in correct order else false. """ refs = self.get_package_refs(package) if refs: takes = [ ref.get(RESIDREF) for ref in refs if ref.get(SEQUENCE) < sequence ] # elastic filter for the archive resource filters out the published items archive_service = get_resource_service(ARCHIVE) query = { 'query': { 'filtered': { 'filter': { 'terms': { '_id': takes } } } } } request = ParsedRequest() request.args = {'source': json.dumps(query)} items = archive_service.get(req=request, lookup=None) return items.count() == 0 return True
def get(self, req, lookup): """ Search for organisation matching the passed q parameter :param req: :param lookup: :return: List of matching organisation strings """ new_req = ParsedRequest() q_str = 'organisation:' + '* organisation:'.join( req.args.get('q', '').split()) + '*' new_req.args = { 'q': q_str, 'default_operator': 'AND', 'projections': '{"organisation": 1}' } ret = super().get(new_req, lookup) # Remove any duplicate entries from the response orgs = [] de_duped = [] for d in ret.docs: if d.get('organisation') not in orgs: orgs.append(d.get('organisation')) de_duped.append(d) ret.docs = de_duped ret.hits['hits']['total'] = len(ret.docs) return ret
def test_args_filter(self): with self.app.app_context(): self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}]) req = ParsedRequest() req.args = {} req.args['filter'] = json.dumps({'term': {'uri': 'foo'}}) self.assertEqual(1, self.app.data.find('items', req, None).count())
def _find_other_sluglines(self, family_id, slugline, versioncreated, desk_id): """ This function given a family_id will return a tuple with the first value true if there is a more recent story in the family, the second value in the tuple will be a list of any sluglines that might exist for the family that are different to the one passed. :param family_id: :param slugline: :param versioncreated: :param desk_id: :return: A tuple as described above """ older_sluglines = [] req = ParsedRequest() query = { "query": { "filtered": {"filter": {"and": [{"term": {"family_id": family_id}}, {"term": {"task.desk": desk_id}}]}} } } req.args = {"source": json.dumps(query), "aggregations": 0} family = superdesk.get_resource_service("published").get(req=req, lookup=None) for member in family: member_slugline = self._get_slugline_with_legal(member) if member_slugline.lower() != slugline.lower(): if member.get("versioncreated") < versioncreated: if member_slugline not in older_sluglines: older_sluglines.append(member_slugline) else: return (True, []) return (False, older_sluglines)
def test_remove_with_query(self): with self.app.app_context(): self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}]) self.app.data.remove('items', {'query': {'term': {'uri': 'bar'}}}) req = ParsedRequest() req.args = {} self.assertEqual(1, self.app.data.find('items', req, None).count())
def test_resource_aggregates(self): with self.app.app_context(): self.app.data.insert('items_with_description', [{ 'uri': 'foo1', 'description': 'test', 'name': 'foo' }]) self.app.data.insert('items_with_description', [{ 'uri': 'foo2', 'description': 'test1', 'name': 'foo' }]) self.app.data.insert('items_with_description', [{ 'uri': 'foo3', 'description': 'test2', 'name': 'foo' }]) self.app.data.insert('items_with_description', [{ 'uri': 'bar1', 'description': 'test3', 'name': 'bar' }]) req = ParsedRequest() req.args = {} response = {} item1 = self.app.data.find('items_with_description', req, {'name': 'foo'}) item2 = self.app.data.find('items_with_description', req, {'name': 'bar'}) item1.extra(response) self.assertEqual(3, item1.count()) self.assertEqual(1, item2.count()) self.assertEqual( 3, response['_aggregations']['type']['buckets'][0]['doc_count'])
def test_resource_filter(self): with self.app.app_context(): self.app.data.insert('items_with_description', [{ 'uri': 'foo', 'description': 'test' }, { 'uri': 'bar' }]) req = ParsedRequest() req.args = {} req.args['source'] = json.dumps( {'query': { 'filtered': { 'filter': { 'term': { 'uri': 'bar' } } } }}) self.assertEqual( 0, self.app.data.find('items_with_description', req, None).count())
def get_archive_items_for_assignment(assignment_id, descending_rewrite_seq=True): if not assignment_id: return [] req = ParsedRequest() req.args = MultiDict() must_not = [{'term': {'state': 'spiked'}}] must = [{'term': {'assignment_id': str(assignment_id)}}, {'term': {'type': 'text'}}] query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': must, 'must_not': must_not } } } } } query['sort'] = [{'rewrite_sequence': 'desc' if descending_rewrite_seq else 'asc'}] query['size'] = 200 req.args['source'] = json.dumps(query) req.args['repo'] = 'archive,published,archived' return list(get_resource_service('search').get(req, None))
def _get_changed_items(self, existing_items, package): """Returns the added and removed items from existing_items :param existing_items: Existing list :param updates: Changes :return: list of removed items and list of added items """ published_service = get_resource_service('published') req = ParsedRequest() query = {'query': {'filtered': {'filter': {'and': [{'terms': {QUEUE_STATE: [ PUBLISH_STATE.QUEUED, PUBLISH_STATE.QUEUED_NOT_TRANSMITTED]}}, {'term': {'item_id': package['item_id']}}]}}}, 'sort': [{'publish_sequence_no': 'desc'}]} req.args = {'source': json.dumps(query)} req.max_results = 1 previously_published_packages = published_service.get(req=req, lookup=None) if not previously_published_packages.count(): return [], [] previously_published_package = previously_published_packages[0] if 'groups' in previously_published_package: old_items = self.package_service.get_residrefs(previously_published_package) added_items = list(set(existing_items) - set(old_items)) removed_items = list(set(old_items) - set(existing_items)) return removed_items, added_items else: return [], []
def test_using_repo_request_attribute(self): with self.app.app_context(): req = ParsedRequest() req.args = {"repo": "ingest"} docs = self.app.data.find("search", req, None) self.assertEquals(1, docs.count()) self.assertEquals("ingest", docs[0]["_type"])
def get_items(self, query): """Return the result of the item search by the given query """ request = ParsedRequest() request.args = {'source': json.dumps(query), 'repo': 'published'} with aggregations_manager([items_over_days_aggregation]): return get_resource_service('search').get(req=request, lookup=None)
def _get_events_and_planning(self, request): """Get list of event and planning based on the search criteria :param request: object representing the HTTP request """ filters, must, must_not = self._get_query(request) page = request.page or 1 max_results = self._get_page_size(request) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': { 'bool': { 'must': must, 'must_not': must_not } }, 'filter': filters, 'size': int((5 * max_results) * math.ceil(page / 3)), 'sort': self._get_sort() }) req.args['projections'] = json.dumps(['_id', 'type', 'event_item']) req.exec_on_fetched_resource = False # don't call on_fetched_resource return get_resource_service('planning_search').get(req=req, lookup=None)
def can_format(self, format_type, article): if format_type != 'AAP SMS' or article[ITEM_TYPE] != CONTENT_TYPE.TEXT \ or article.get(ITEM_STATE, '') == CONTENT_STATE.KILLED \ or not article.get('flags', {}).get('marked_for_sms', False): return False # need to check that a story with the same sms_message has not been published to SMS before query = {"query": { "filtered": { "filter": { "and": [ {"term": {"state": CONTENT_STATE.PUBLISHED}}, {"term": {"sms_message": article.get('sms_message', article.get('abstract', ''))}}, {"term": {"flags.marked_for_sms": True}}, {"not": {"term": {"queue_state": "in_progress"}}} ] } } } } req = ParsedRequest() req.args = {'source': json.dumps(query)} published = superdesk.get_resource_service('published').get(req=req, lookup=None) if published and published.count(): return False return article.get('flags', {}).get('marked_for_sms', False)
def get(self, req, lookup): """ Return a list of items related to the given item. The given item id is retrieved from the lookup dictionary as 'item_id' """ if 'item_id' not in lookup: raise SuperdeskApiError.badRequestError('The item identifier is required') item = get_resource_service('archive_autosave').find_one(req=None, _id=lookup['item_id']) if not item: item = get_resource_service('archive').find_one(req=None, _id=lookup['item_id']) if not item: raise SuperdeskApiError.notFoundError('Invalid item identifer') keywords = self.provider.get_keywords(self._transform(item)) if not keywords: return ElasticCursor([]) query = { 'query': { 'filtered': { 'query': { 'query_string': { 'query': ' '.join(kwd['text'] for kwd in keywords) } } } } } req = ParsedRequest() req.args = {'source': json.dumps(query), 'repo': 'archive,published,archived'} return get_resource_service('search').get(req=req, lookup=None)
def test_using_repo_request_attribute(self): with self.app.app_context(): req = ParsedRequest() req.args = {"repo": "ingest"} docs = self.app.data.find("search", req, None)[0] self.assertEquals(1, docs.count()) self.assertEquals("ingest", docs[0]["_type"])
def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal): """Get the pulished items where flag is moved. :param list item_ids: List of item :param bool move_to_legal: move_to_legal boolean flag :return: list of published items """ if item_ids: try: query = { 'query': { 'filtered': { 'filter': { 'and': [ {'terms': {'item_id': item_ids}}, {'term': {'moved_to_legal': move_to_legal}} ] } } } } request = ParsedRequest() request.args = {'source': json.dumps(query)} return list(super().get(req=request, lookup=None)) except: logger.exception('Failed to get published items ' 'by moved to legal: {} -- ids: {}.'.format(move_to_legal, item_ids)) return []
def _get_broadcast_items(self, ids, include_archived_repo=False): """Returns list of broadcast items. Get the broadcast items for the master_id :param list ids: list of item ids :param include_archived_repo True if archived repo needs to be included in search, default is False :return list: list of broadcast items """ query = { "query": { "bool": { "filter": [ { "term": { "genre.name": BROADCAST_GENRE } }, { "terms": { "broadcast.master_id": ids } }, ], } } } req = ParsedRequest() repos = "archive,published" if include_archived_repo: repos = "archive,published,archived" req.args = {"source": json.dumps(query), "repo": repos} return get_resource_service("search").get(req=req, lookup=None)
def test_compare_repos(self): with self.app.app_context(): req = ParsedRequest() req.args = {} req.max_results = 25 data = [{'headline': 'test {}'.format(i), 'slugline': 'rebuild {}'.format(i), 'type': 'text' if (i % 2 == 0) else 'picture'} for i in range(1, 100)] resolve_document_etag(data, 'archive') superdesk.app.data._search_backend('archive').bulk_insert('archive', data) get_resource_service('archive').post(data) consistency_init(self.app) items = get_resource_service('archive').get(req, {}) self.assertEquals(99, items.count()) consistency_record = CompareRepositories().run('archive', self.app.config['ELASTICSEARCH_URL'], self.app.config['ELASTICSEARCH_INDEXES'].get('archive')) self.assertEquals(consistency_record['mongo'], 99) self.assertEquals(consistency_record['elastic'], 198) self.assertEquals(consistency_record['identical'], 99) self.assertEquals(consistency_record['mongo_only'], 0) self.assertEquals(consistency_record['elastic_only'], 99) self.assertEquals(consistency_record['inconsistent'], 0)
def _get_broadcast_items(self, ids, include_archived_repo=False): """Returns list of broadcast items. Get the broadcast items for the master_id :param list ids: list of item ids :param include_archived_repo True if archived repo needs to be included in search, default is False :return list: list of broadcast items """ query = { 'query': { 'bool': { 'filter': [ { 'term': { 'genre.name': BROADCAST_GENRE } }, { 'terms': { 'broadcast.master_id': ids } }, ], } } } req = ParsedRequest() repos = 'archive,published' if include_archived_repo: repos = 'archive,published,archived' req.args = {'source': json.dumps(query), 'repo': repos} return get_resource_service('search').get(req=req, lookup=None)
def filter_subscribers(self, doc, subscribers, target_media_type): """ Filter subscribers to whom the current story is going to be delivered. """ filtered_subscribers = [] req = ParsedRequest() req.args = {'is_global': True} service = get_resource_service('publish_filters') global_filters = list(service.get(req=req, lookup=None)) for subscriber in subscribers: if target_media_type: can_send_takes_packages = subscriber['subscriber_type'] == SUBSCRIBER_TYPES.DIGITAL if target_media_type == WIRE and can_send_takes_packages or target_media_type == DIGITAL \ and not can_send_takes_packages: continue if doc.get('targeted_for'): matching_target = [t for t in doc.get('targeted_for') if t['name'] == subscriber.get('subscriber_type', '') or t['name'] == subscriber.get('geo_restrictions', '')] if len(matching_target) > 0 and matching_target[0]['allow'] is False: continue if not self.conforms_global_filter(subscriber, global_filters, doc): continue if not self.conforms_publish_filter(subscriber, doc): continue filtered_subscribers.append(subscriber) return filtered_subscribers
def test_versions_across_collections_after_publish(self): self.app.data.insert('archive_versions', self.article_versions) # Publishing an Article doc = self.articles[3] original = doc.copy() published_version_number = original[config.VERSION] + 1 get_resource_service(ARCHIVE_PUBLISH).patch(id=doc[config.ID_FIELD], updates={ITEM_STATE: CONTENT_STATE.PUBLISHED, config.VERSION: published_version_number}) article_in_production = get_resource_service(ARCHIVE).find_one(req=None, _id=original[config.ID_FIELD]) self.assertIsNotNone(article_in_production) self.assertEqual(article_in_production[ITEM_STATE], CONTENT_STATE.PUBLISHED) self.assertEqual(article_in_production[config.VERSION], published_version_number) enqueue_published() lookup = {'item_id': original[config.ID_FIELD], 'item_version': published_version_number} queue_items = list(get_resource_service(PUBLISH_QUEUE).get(req=None, lookup=lookup)) assert len(queue_items) > 0, \ "Transmission Details are empty for published item %s" % original[config.ID_FIELD] lookup = {'item_id': original[config.ID_FIELD], config.VERSION: published_version_number} request = ParsedRequest() request.args = {'aggregations': 0} items_in_published_collection = list(get_resource_service(PUBLISHED).get(req=request, lookup=lookup)) assert len(items_in_published_collection) > 0, \ "Item not found in published collection %s" % original[config.ID_FIELD]
def get_rewritten_take_packages_per_event(self, event_id): """ Returns all the published and rewritten take stories for the same event """ try: query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'package_type': 'takes' } }, { 'term': { 'event_id': event_id } }, { 'exists': { 'field': 'rewritten_by' } }] } } } } } request = ParsedRequest() request.args = {'source': json.dumps(query)} return super().get(req=request, lookup=None) except: return []
def test_versions_across_collections_after_publish(self): self.app.data.insert('archive_versions', self.article_versions) # Publishing an Article doc = self.articles[7] original = doc.copy() published_version_number = original[config.VERSION] + 1 get_resource_service(ARCHIVE_PUBLISH).patch(id=doc[config.ID_FIELD], updates={ITEM_STATE: CONTENT_STATE.PUBLISHED, config.VERSION: published_version_number}) article_in_production = get_resource_service(ARCHIVE).find_one(req=None, _id=original[config.ID_FIELD]) self.assertIsNotNone(article_in_production) self.assertEqual(article_in_production[ITEM_STATE], CONTENT_STATE.PUBLISHED) self.assertEqual(article_in_production[config.VERSION], published_version_number) enqueue_published() lookup = {'item_id': original[config.ID_FIELD], 'item_version': published_version_number} queue_items = list(get_resource_service(PUBLISH_QUEUE).get(req=None, lookup=lookup)) assert len(queue_items) > 0, \ "Transmission Details are empty for published item %s" % original[config.ID_FIELD] lookup = {'item_id': original[config.ID_FIELD], config.VERSION: published_version_number} request = ParsedRequest() request.args = {'aggregations': 0} items_in_published_collection = list(get_resource_service(PUBLISHED).get(req=request, lookup=lookup)) assert len(items_in_published_collection) > 0, \ "Item not found in published collection %s" % original[config.ID_FIELD]
def test_remove_by_id(self): with self.app.app_context(): self.ids = self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}]) self.app.data.remove('items', {'_id': self.ids[0]}) req = ParsedRequest() req.args = {} self.assertEqual(1, self.app.data.find('items', req, None).count())
def test_using_repo_request_attribute(self): with self.app.app_context(): req = ParsedRequest() req.args = {'repo': 'ingest'} docs = self.app.data.find('search', req, None)[0] self.assertEquals(1, docs.count()) self.assertEquals('ingest', docs[0]['_type'])
def get_product_items(self, product_id, size): query = _items_query() product = get_resource_service('products').find_one(req=None, _id=product_id) if not product: return query['bool']['should'] = [] get_resource_service('section_filters').apply_section_filter( query, product.get('product_type')) if product.get('sd_product_id'): query['bool']['should'].append( {'term': { 'products.code': product['sd_product_id'] }}) if product.get('query'): query['bool']['should'].append(query_string(product['query'])) query['bool']['minimum_should_match'] = 1 source = {'query': query} source['sort'] = [{'versioncreated': 'desc'}] source['size'] = size source['from'] = 0 source['post_filter'] = {'bool': {'must': []}} internal_req = ParsedRequest() internal_req.args = {'source': json.dumps(source)} return list(super().get(internal_req, None))
def _get_combined_view_data(self, items, request): """Get list of event and planning for the combined view :param items: :param request: object representing the HTTP request """ ids = set() for item in items: # don't want related planing items _id = item.get('event_item') or item.get('_id') ids.add(_id) filters = self._get_date_filters(request) page = request.page or 1 page_size = self._get_page_size(request) req = ParsedRequest() req.args = MultiDict() req.args['source'] = json.dumps({ 'query': { 'bool': { 'must': [{'terms': {'_id': list(ids)}}], } }, 'filter': filters, 'sort': self._get_sort(), 'size': self._get_page_size(request), 'from': (page - 1) * page_size }) req.page = request.page or 1 req.max_results = self._get_page_size(request) return get_resource_service('planning_search').get(req=req, lookup=None)
def test_using_repo_request_attribute(self): with self.app.app_context(): req = ParsedRequest() req.args = {'repo': 'ingest'} docs = self.app.data.find('search', req, None) self.assertEquals(1, docs.count()) self.assertEquals('ingest', docs[0]['_type'])
def test_compare_repos(self): with self.app.app_context(): req = ParsedRequest() req.args = {} req.max_results = 25 data = [{ 'headline': 'test {}'.format(i), 'slugline': 'rebuild {}'.format(i), 'type': 'text' if (i % 2 == 0) else 'picture' } for i in range(1, 100)] resolve_document_etag(data, 'archive') superdesk.app.data._search_backend('archive').bulk_insert( 'archive', data) get_resource_service('archive').post(data) consistency_init(self.app) items = get_resource_service('archive').get(req, {}) self.assertEquals(99, items.count()) consistency_record = CompareRepositories().run( 'archive', self.app.config['ELASTICSEARCH_URL'], self.app.config['ELASTICSEARCH_INDEXES'].get('archive')) self.assertEquals(consistency_record['mongo'], 99) self.assertEquals(consistency_record['elastic'], 198) self.assertEquals(consistency_record['identical'], 99) self.assertEquals(consistency_record['mongo_only'], 0) self.assertEquals(consistency_record['elastic_only'], 99) self.assertEquals(consistency_record['inconsistent'], 0)
def get_items(self, item_ids): try: query = { 'bool': { 'must_not': [ { 'term': { 'type': 'composite' } }, ], 'must': [{ 'terms': { '_id': item_ids } }], } } source = {'query': query} source['size'] = len(item_ids) req = ParsedRequest() req.args = {'source': json.dumps(source)} return super().get(req, None) except Exception as exc: logger.error('Error in get_items for query: {}'.format( json.dumps(source)), exc, exc_info=True)
def get_rewritten_items_by_event_story(self, event_id, rewrite_id): """ Returns all the published and rewritten stories for the given event and rewrite_id""" try: query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': [{ 'term': { 'event_id': event_id } }, { 'term': { 'rewritten_by': rewrite_id } }] } } } } } request = ParsedRequest() request.args = {'source': json.dumps(query)} return super().get(req=request, lookup=None) except: return []
def _get_broadcast_items(self, ids): """ Get the broadcast items for the master_id and takes_package_id :param list ids: list of item ids :return list: list of broadcast items """ query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': {'term': {'genre.name': BROADCAST_GENRE}}, 'should': [ {'terms': {'broadcast.master_id': ids}}, {'terms': {'broadcast.takes_package_id': ids}} ] } } } } } req = ParsedRequest() req.args = {'source': json.dumps(query), 'repo': 'archive,published'} return get_resource_service('search').get(req=req, lookup=None)
def test_search_capi_aggregations(self): self.content_api.publish( { '_id': '1', 'guid': '1', 'type': 'text', 'anpa_category': [{ 'qcode': 'i', 'name': 'International News' }], 'headline': 'Man bites dog', 'source': 'AAA', 'urgency': 1 }, []) self.content_api.publish( { '_id': '2', 'guid': '2', 'type': 'text', 'anpa_category': [{ 'qcode': 'i', 'name': 'International News' }], 'headline': 'Man bites cat', 'source': 'BBB', 'urgency': 2 }, []) test = superdesk.get_resource_service('search_capi') req = ParsedRequest() req.args = MultiDict([('aggregations', 1)]) resp = test.get(req=req, lookup=None) self.assertEqual( resp.hits['aggregations']['category']['buckets'][0]['doc_count'], 2)
def assert_products_query(user_id, args=None, products=None): with app.test_request_context(): session['user'] = user_id search = SearchQuery() if args is None: service.prefill_search_query(search) else: req = ParsedRequest() req.args = args service.prefill_search_query(search, req) service.apply_products_filter(search) for product in products: if product.get('query'): assert { 'query_string': { 'query': product['query'], 'default_operator': 'AND', 'lenient': True } } in search.query['bool']['should'] sd_product_ids = [ product['sd_product_id'] for product in products if product.get('sd_product_id') ] if len(sd_product_ids): assert { 'terms': { 'products.code': sd_product_ids } } in search.query['bool']['should']
def get_last_published_version(self, _id): """Returns the last published entry for the passed item id :param _id: :return: """ try: query = { "query": { "filtered": { "filter": { "bool": { "must": [{ "term": { "item_id": _id } }, { "term": { LAST_PUBLISHED_VERSION: True } }] } } } } } request = ParsedRequest() request.args = {"source": json.dumps(query), "repo": "published"} items = list(self.get(req=request, lookup=None)) if items: return items[0] except Exception: return None
def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal): """Get the pulished items where flag is moved. :param list item_ids: List of item :param bool move_to_legal: move_to_legal boolean flag :return: list of published items """ if item_ids: try: query = { "query": { "filtered": { "filter": { "and": [{"terms": {"item_id": item_ids}}, {"term": {"moved_to_legal": move_to_legal}}] } } } } request = ParsedRequest() request.args = {"source": json.dumps(query)} return list(super().get(req=request, lookup=None)) except: logger.exception( "Failed to get published items " "by moved to legal: {} -- ids: {}.".format(move_to_legal, item_ids) ) return []
def _enchance_assignment(self, doc): """Populate `item_ids` with ids for all linked Archive items for an Assignment Using the `search` resource service, retrieve the list of Archive items linked to the provided Assignment. """ query = { 'query': { 'filtered': { 'filter': { 'bool': { 'must': { 'term': { 'assignment_id': str(doc[config.ID_FIELD]) } }, } } } } } req = ParsedRequest() repos = 'archive,published,archived' req.args = {'source': json.dumps(query), 'repo': repos} items = list(get_resource_service('search').get(req=req, lookup=None)) if items: doc['item_ids'] = [ str(item.get(config.ID_FIELD)) for item in items ] self.set_type(doc, doc)
def test_resource_filter(self): with self.app.app_context(): self.app.data.insert('items_with_description', [{'uri': 'foo', 'description': 'test'}, {'uri': 'bar'}]) req = ParsedRequest() req.args = {} req.args['source'] = json.dumps({'query': {'filtered': {'filter': {'term': {'uri': 'bar'}}}}}) self.assertEqual(0, self.app.data.find('items_with_description', req, None).count())
def test_sub_resource_lookup_with_schema_filter(self): with self.app.app_context(): self.app.data.insert('items_with_description', [{'uri': 'foo', 'description': 'test', 'name': 'foo'}]) req = ParsedRequest() req.args = {} self.assertEqual(1, self.app.data.find('items_with_description', req, {'name': 'foo'}).count()) self.assertEqual(0, self.app.data.find('items_with_description', req, {'name': 'bar'}).count())
def test_sub_resource_lookup(self): with self.app.app_context(): self.app.data.insert('items', [{'uri': 'foo', 'name': 'foo'}]) req = ParsedRequest() req.args = {} self.assertEqual(1, self.app.data.find('items', req, {'name': 'foo'}).count()) self.assertEqual(0, self.app.data.find('items', req, {'name': 'bar'}).count())
def test_search_capi_filter(self): subscriber = {'_id': 'sub1'} self.content_api.publish({'_id': 'foo', 'guid': 'foo', 'type': 'text', 'anpa_category': [{'qcode': 'i', 'name': 'International News'}], 'headline': 'Man bites dog'}, [subscriber]) self.content_api.publish({'_id': 'bar', 'guid': 'bar', 'anpa_category': [{'qcode': 'i', 'name': 'International News'}], 'type': 'text'}, [{'_id': 'sub2'}]) self.content_api.publish({'_id': 'nat', 'guid': 'nat', 'anpa_category': [{'qcode': 'a', 'name': 'National News'}], 'type': 'text'}, [{'_id': 'sub2'}]) test = superdesk.get_resource_service('search_capi') req = ParsedRequest() req.args = MultiDict([('filter', '[{"term": {"service.code": "i"}}]')]) resp = test.get(req=req, lookup=None) self.assertEqual(resp.count(), 2) self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'i') self.assertEqual(resp.docs[1].get('anpa_category')[0].get('qcode'), 'i') req = ParsedRequest() req.args = MultiDict([('service', 'i')]) resp = test.get(req=req, lookup=None) self.assertEqual(resp.count(), 2) self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'i') self.assertEqual(resp.docs[1].get('anpa_category')[0].get('qcode'), 'i') req = ParsedRequest() req.args = MultiDict([('service', '["a"]')]) resp = test.get(req=req, lookup=None) self.assertEqual(resp.count(), 1) self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'a') req = ParsedRequest() req.args = MultiDict([('service', 'i'), ('subscribers', 'sub1')]) resp = test.get(req=req, lookup=None) self.assertEqual(resp.count(), 1) self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'i') self.assertEqual(resp.docs[0].get('subscribers')[0], 'sub1') req = ParsedRequest() req.args = MultiDict([('subscribers', 'sub2')]) resp = test.get(req=req, lookup=None) self.assertEqual(resp.count(), 2) self.assertEqual(resp.docs[0].get('subscribers')[0], 'sub2') self.assertEqual(resp.docs[1].get('subscribers')[0], 'sub2')
def get_other_published_items(self, _id): try: query = {"query": {"filtered": {"filter": {"term": {"item_id": _id}}}}} request = ParsedRequest() request.args = {"source": json.dumps(query)} return super().get(req=request, lookup=None) except: return []
def get_other_published_items(self, _id): try: query = {'query': {'filtered': {'filter': {'term': {'item_id': _id}}}}} request = ParsedRequest() request.args = {'source': json.dumps(query)} return super().get(req=request, lookup=None) except: return []