def test_should_highlight(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {'es_highlight': 1}
self.assertTrue(self.app.data.should_highlight(req))
req.args = {'es_highlight': '0'}
self.assertFalse(self.app.data.should_highlight(req))
def test_should_aggregate(self):
with self.app.app_context():
self.app.config['ELASTICSEARCH_AUTO_AGGREGATIONS'] = False
req = ParsedRequest()
req.args = {'aggregations': 1}
self.assertTrue(self.app.data.should_aggregate(req))
req.args = {'aggregations': '0'}
self.assertFalse(self.app.data.should_aggregate(req))
def get_expired_items(self, page_size):
"""Get expired item that are not moved to legal
:return:
"""
query = {
"query": {
"filtered": {
"filter": {
"and": [
{"range": {"expiry": {"lt": "now"}}},
{"term": {"moved_to_legal": False}},
{"not": {"term": {"state": CONTENT_STATE.SCHEDULED}}},
]
}
}
}
}
service = get_resource_service("published")
req = ParsedRequest()
req.args = {"source": json.dumps(query)}
req.sort = '[("publish_sequence_no", 1)]'
cursor = service.get(req=req, lookup=None)
count = cursor.count()
no_of_pages = 0
if count:
no_of_pages = len(range(0, count, page_size))
sequence_no = cursor[0]["publish_sequence_no"]
logger.info("Number of items to move to legal archive: {}, pages={}".format(count, no_of_pages))
for page in range(0, no_of_pages):
logger.info(
"Fetching published items " "for page number: {} sequence no: {}".format((page + 1), sequence_no)
)
req = ParsedRequest()
page_query = deepcopy(query)
sequence_filter = {"range": {"publish_sequence_no": {"gte": sequence_no}}}
if page == 0:
sequence_filter = {"range": {"publish_sequence_no": {"gte": sequence_no}}}
else:
sequence_filter = {"range": {"publish_sequence_no": {"gt": sequence_no}}}
page_query["query"]["filtered"]["filter"]["and"].append(sequence_filter)
req.args = {"source": json.dumps(page_query)}
req.sort = '[("publish_sequence_no", 1)]'
req.max_results = page_size
cursor = service.get(req=req, lookup=None)
items = list(cursor)
if len(items):
sequence_no = items[len(items) - 1]["publish_sequence_no"]
logger.info(
"Fetched No. of Items: {} for page: {} " "For import into legal archive.".format(len(items), (page + 1))
)
yield items
def test_should_project(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {'projections': json.dumps(["priority", "urgency", "word_count", "slugline", "highlights"])}
self.assertTrue(self.app.data.should_project(req))
req.args = {'projections': json.dumps([])}
self.assertFalse(self.app.data.should_project(req))
req.args = {}
self.assertFalse(self.app.data.should_project(req))
def get_expired_items(self, page_size):
"""Get expired item that are not moved to legal
:return:
"""
query = {
'query': {
'filtered': {
'filter': {
'and': [
{'range': {'expiry': {'lt': 'now'}}},
{'term': {'moved_to_legal': False}},
{'not': {'term': {'state': CONTENT_STATE.SCHEDULED}}}
]
}
}
}
}
service = get_resource_service('published')
req = ParsedRequest()
req.args = {'source': json.dumps(query)}
req.sort = '[("publish_sequence_no", 1)]'
cursor = service.get(req=req, lookup=None)
count = cursor.count()
no_of_pages = 0
if count:
no_of_pages = len(range(0, count, page_size))
sequence_no = cursor[0]['publish_sequence_no']
logger.info('Number of items to move to legal archive: {}, pages={}'.format(count, no_of_pages))
for page in range(0, no_of_pages):
logger.info('Fetching published items '
'for page number: {} sequence no: {}'. format((page + 1), sequence_no))
req = ParsedRequest()
page_query = deepcopy(query)
sequence_filter = {'range': {'publish_sequence_no': {'gte': sequence_no}}}
if page == 0:
sequence_filter = {'range': {'publish_sequence_no': {'gte': sequence_no}}}
else:
sequence_filter = {'range': {'publish_sequence_no': {'gt': sequence_no}}}
page_query['query']['filtered']['filter']['and'].append(sequence_filter)
req.args = {'source': json.dumps(page_query)}
req.sort = '[("publish_sequence_no", 1)]'
req.max_results = page_size
cursor = service.get(req=req, lookup=None)
items = list(cursor)
if len(items):
sequence_no = items[len(items) - 1]['publish_sequence_no']
logger.info('Fetched No. of Items: {} for page: {} '
'For import into legal archive.'.format(len(items), (page + 1)))
yield items
def test_resource_aggregates_no_auto(self):
with self.app.app_context():
self.app.data.insert('items_with_description', [{'uri': 'foo'}])
self.app.config['ELASTICSEARCH_AUTO_AGGREGATIONS'] = False
req = ParsedRequest()
req.args = {}
response = {}
cursor = self.app.data.find('items_with_description', req, {})
cursor.extra(response)
self.assertNotIn('_aggregations', response)
req.args = {'aggregations': 1}
cursor = self.app.data.find('items_with_description', req, {})
cursor.extra(response)
self.assertIn('_aggregations', response)
def test_aggs_with_request(self):
params = {
'params': {'dates': {'filter': 'yesterday'}},
'aggs': {
'group': {'field': 'anpa_category.qcode'},
'subgroup': {'field': 'urgency'}
},
'repo': 'published',
'return_type': 'text/csv'
}
expected_args = {
'params': params['params'],
'aggs': params['aggs'],
'repo': params['repo'],
'return_type': params['return_type']
}
request = ParsedRequest()
# Request object with source as a json in string format
request.args = {
'params': json.dumps(params['params']),
'repo': params['repo'],
'return_type': params['return_type'],
'aggs': json.dumps(params['aggs'])
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with args an ImmutableMultiDict
request.args = ImmutableMultiDict({
'params': params['params'],
'repo': params['repo'],
'return_type': params['return_type'],
'aggs': params['aggs']
})
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
# Request object with source as a normal dict
request.args = {
'params': params['params'],
'repo': params['repo'],
'return_type': params['return_type'],
'aggs': params['aggs']
}
args = self.service._get_request_or_lookup(req=request, lookup=None)
self.assertEqual(args, expected_args)
def _get_broadcast_items(self, ids, include_archived_repo=False):
"""
Get the broadcast items for the master_id and takes_package_id
:param list ids: list of item ids
:param include_archived_repo True if archived repo needs to be included in search, default is False
:return list: list of broadcast items
"""
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': {'term': {'genre.name': BROADCAST_GENRE}},
'should': [
{'terms': {'broadcast.master_id': ids}},
{'terms': {'broadcast.takes_package_id': ids}}
]
}
}
}
}
}
req = ParsedRequest()
repos = 'archive,published'
if include_archived_repo:
repos = 'archive,published,archived'
req.args = {'source': json.dumps(query), 'repo': repos}
return get_resource_service('search').get(req=req, lookup=None)
def get_publish_items(item_id, last_version):
query = {'query': {'filtered': {'filter': {'and': [
{'term': {'item_id': item_id}}, {'term': {LAST_PUBLISHED_VERSION: last_version}}
]}}}}
request = ParsedRequest()
request.args = {'source': json.dumps(query), 'aggregations': 0}
return self.app.data.find(PUBLISHED, req=request, lookup=None)
def get_publish_items(item_id, last_version):
query = {'query': {'filtered': {'filter': {'and': [
{'term': {'item_id': item_id}}, {'term': {LAST_PUBLISHED_VERSION: last_version}}
]}}}}
request = ParsedRequest()
request.args = {'source': json.dumps(query), 'aggregations': 0}
return self.app.data.find(PUBLISHED, req=request, lookup=None)
def get_elastic_item(self, id):
resource = superdesk.get_resource_service(self.resource_name)
query = {'query': {'filtered': {'filter': {'term': {'_id': id}}}}}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
items = resource.get(req=request, lookup=None)
return items[0]
def get_last_published_version(self, _id):
"""Returns the last published entry for the passed item id
:param _id:
:return:
"""
try:
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'item_id': _id
}
}, {
'term': {
LAST_PUBLISHED_VERSION: True
}
}]
}
}
}
}
}
request = ParsedRequest()
request.args = {'source': json.dumps(query), 'repo': 'published'}
items = list(self.get(req=request, lookup=None))
if items:
return items[0]
except Exception:
return None
def can_publish_take(self, package, sequence):
"""
Takes can be published only in ascending order. This function check that if there are any
unpublished takes before the current take.
:param dict package: takes packages
:param int sequence: take sequence of the published take
:return: True if takes are published in correct order else false.
"""
refs = self.get_package_refs(package)
if refs:
takes = [
ref.get(RESIDREF) for ref in refs
if ref.get(SEQUENCE) < sequence
]
# elastic filter for the archive resource filters out the published items
archive_service = get_resource_service(ARCHIVE)
query = {
'query': {
'filtered': {
'filter': {
'terms': {
'_id': takes
}
}
}
}
}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
items = archive_service.get(req=request, lookup=None)
return items.count() == 0
return True
def get(self, req, lookup):
"""
Search for organisation matching the passed q parameter
:param req:
:param lookup:
:return: List of matching organisation strings
"""
new_req = ParsedRequest()
q_str = 'organisation:' + '* organisation:'.join(
req.args.get('q', '').split()) + '*'
new_req.args = {
'q': q_str,
'default_operator': 'AND',
'projections': '{"organisation": 1}'
}
ret = super().get(new_req, lookup)
# Remove any duplicate entries from the response
orgs = []
de_duped = []
for d in ret.docs:
if d.get('organisation') not in orgs:
orgs.append(d.get('organisation'))
de_duped.append(d)
ret.docs = de_duped
ret.hits['hits']['total'] = len(ret.docs)
return ret
def test_args_filter(self):
with self.app.app_context():
self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}])
req = ParsedRequest()
req.args = {}
req.args['filter'] = json.dumps({'term': {'uri': 'foo'}})
self.assertEqual(1, self.app.data.find('items', req, None).count())
def _find_other_sluglines(self, family_id, slugline, versioncreated, desk_id):
"""
This function given a family_id will return a tuple with the first value true if there is
a more recent story in the family, the second value in the tuple will be a list of any sluglines
that might exist for the family that are different to the one passed.
:param family_id:
:param slugline:
:param versioncreated:
:param desk_id:
:return: A tuple as described above
"""
older_sluglines = []
req = ParsedRequest()
query = {
"query": {
"filtered": {"filter": {"and": [{"term": {"family_id": family_id}}, {"term": {"task.desk": desk_id}}]}}
}
}
req.args = {"source": json.dumps(query), "aggregations": 0}
family = superdesk.get_resource_service("published").get(req=req, lookup=None)
for member in family:
member_slugline = self._get_slugline_with_legal(member)
if member_slugline.lower() != slugline.lower():
if member.get("versioncreated") < versioncreated:
if member_slugline not in older_sluglines:
older_sluglines.append(member_slugline)
else:
return (True, [])
return (False, older_sluglines)
def test_remove_with_query(self):
with self.app.app_context():
self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}])
self.app.data.remove('items', {'query': {'term': {'uri': 'bar'}}})
req = ParsedRequest()
req.args = {}
self.assertEqual(1, self.app.data.find('items', req, None).count())
def test_resource_aggregates(self):
with self.app.app_context():
self.app.data.insert('items_with_description', [{
'uri': 'foo1',
'description': 'test',
'name': 'foo'
}])
self.app.data.insert('items_with_description', [{
'uri': 'foo2',
'description': 'test1',
'name': 'foo'
}])
self.app.data.insert('items_with_description', [{
'uri': 'foo3',
'description': 'test2',
'name': 'foo'
}])
self.app.data.insert('items_with_description', [{
'uri': 'bar1',
'description': 'test3',
'name': 'bar'
}])
req = ParsedRequest()
req.args = {}
response = {}
item1 = self.app.data.find('items_with_description', req,
{'name': 'foo'})
item2 = self.app.data.find('items_with_description', req,
{'name': 'bar'})
item1.extra(response)
self.assertEqual(3, item1.count())
self.assertEqual(1, item2.count())
self.assertEqual(
3,
response['_aggregations']['type']['buckets'][0]['doc_count'])
def test_resource_filter(self):
with self.app.app_context():
self.app.data.insert('items_with_description',
[{
'uri': 'foo',
'description': 'test'
}, {
'uri': 'bar'
}])
req = ParsedRequest()
req.args = {}
req.args['source'] = json.dumps(
{'query': {
'filtered': {
'filter': {
'term': {
'uri': 'bar'
}
}
}
}})
self.assertEqual(
0,
self.app.data.find('items_with_description', req,
None).count())
def get_archive_items_for_assignment(assignment_id, descending_rewrite_seq=True):
if not assignment_id:
return []
req = ParsedRequest()
req.args = MultiDict()
must_not = [{'term': {'state': 'spiked'}}]
must = [{'term': {'assignment_id': str(assignment_id)}},
{'term': {'type': 'text'}}]
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': must,
'must_not': must_not
}
}
}
}
}
query['sort'] = [{'rewrite_sequence': 'desc' if descending_rewrite_seq else 'asc'}]
query['size'] = 200
req.args['source'] = json.dumps(query)
req.args['repo'] = 'archive,published,archived'
return list(get_resource_service('search').get(req, None))
def _get_changed_items(self, existing_items, package):
"""Returns the added and removed items from existing_items
:param existing_items: Existing list
:param updates: Changes
:return: list of removed items and list of added items
"""
published_service = get_resource_service('published')
req = ParsedRequest()
query = {'query': {'filtered': {'filter': {'and': [{'terms': {QUEUE_STATE: [
PUBLISH_STATE.QUEUED, PUBLISH_STATE.QUEUED_NOT_TRANSMITTED]}},
{'term': {'item_id': package['item_id']}}]}}}, 'sort': [{'publish_sequence_no': 'desc'}]}
req.args = {'source': json.dumps(query)}
req.max_results = 1
previously_published_packages = published_service.get(req=req, lookup=None)
if not previously_published_packages.count():
return [], []
previously_published_package = previously_published_packages[0]
if 'groups' in previously_published_package:
old_items = self.package_service.get_residrefs(previously_published_package)
added_items = list(set(existing_items) - set(old_items))
removed_items = list(set(old_items) - set(existing_items))
return removed_items, added_items
else:
return [], []
def test_using_repo_request_attribute(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {"repo": "ingest"}
docs = self.app.data.find("search", req, None)
self.assertEquals(1, docs.count())
self.assertEquals("ingest", docs[0]["_type"])
def get_items(self, query):
"""Return the result of the item search by the given query
"""
request = ParsedRequest()
request.args = {'source': json.dumps(query), 'repo': 'published'}
with aggregations_manager([items_over_days_aggregation]):
return get_resource_service('search').get(req=request, lookup=None)
def _get_events_and_planning(self, request):
"""Get list of event and planning based on the search criteria
:param request: object representing the HTTP request
"""
filters, must, must_not = self._get_query(request)
page = request.page or 1
max_results = self._get_page_size(request)
req = ParsedRequest()
req.args = MultiDict()
req.args['source'] = json.dumps({
'query': {
'bool': {
'must': must,
'must_not': must_not
}
},
'filter':
filters,
'size':
int((5 * max_results) * math.ceil(page / 3)),
'sort':
self._get_sort()
})
req.args['projections'] = json.dumps(['_id', 'type', 'event_item'])
req.exec_on_fetched_resource = False # don't call on_fetched_resource
return get_resource_service('planning_search').get(req=req,
lookup=None)
def can_format(self, format_type, article):
if format_type != 'AAP SMS' or article[ITEM_TYPE] != CONTENT_TYPE.TEXT \
or article.get(ITEM_STATE, '') == CONTENT_STATE.KILLED \
or not article.get('flags', {}).get('marked_for_sms', False):
return False
# need to check that a story with the same sms_message has not been published to SMS before
query = {"query": {
"filtered": {
"filter": {
"and": [
{"term": {"state": CONTENT_STATE.PUBLISHED}},
{"term": {"sms_message": article.get('sms_message', article.get('abstract', ''))}},
{"term": {"flags.marked_for_sms": True}},
{"not": {"term": {"queue_state": "in_progress"}}}
]
}
}
}
}
req = ParsedRequest()
req.args = {'source': json.dumps(query)}
published = superdesk.get_resource_service('published').get(req=req, lookup=None)
if published and published.count():
return False
return article.get('flags', {}).get('marked_for_sms', False)
def get(self, req, lookup):
"""
Return a list of items related to the given item. The given item id is retrieved
from the lookup dictionary as 'item_id'
"""
if 'item_id' not in lookup:
raise SuperdeskApiError.badRequestError('The item identifier is required')
item = get_resource_service('archive_autosave').find_one(req=None, _id=lookup['item_id'])
if not item:
item = get_resource_service('archive').find_one(req=None, _id=lookup['item_id'])
if not item:
raise SuperdeskApiError.notFoundError('Invalid item identifer')
keywords = self.provider.get_keywords(self._transform(item))
if not keywords:
return ElasticCursor([])
query = {
'query': {
'filtered': {
'query': {
'query_string': {
'query': ' '.join(kwd['text'] for kwd in keywords)
}
}
}
}
}
req = ParsedRequest()
req.args = {'source': json.dumps(query), 'repo': 'archive,published,archived'}
return get_resource_service('search').get(req=req, lookup=None)
def test_using_repo_request_attribute(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {"repo": "ingest"}
docs = self.app.data.find("search", req, None)[0]
self.assertEquals(1, docs.count())
self.assertEquals("ingest", docs[0]["_type"])
def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal):
"""Get the pulished items where flag is moved.
:param list item_ids: List of item
:param bool move_to_legal: move_to_legal boolean flag
:return: list of published items
"""
if item_ids:
try:
query = {
'query': {
'filtered': {
'filter': {
'and': [
{'terms': {'item_id': item_ids}},
{'term': {'moved_to_legal': move_to_legal}}
]
}
}
}
}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
return list(super().get(req=request, lookup=None))
except:
logger.exception('Failed to get published items '
'by moved to legal: {} -- ids: {}.'.format(move_to_legal, item_ids))
return []
def _get_broadcast_items(self, ids, include_archived_repo=False):
"""Returns list of broadcast items.
Get the broadcast items for the master_id
:param list ids: list of item ids
:param include_archived_repo True if archived repo needs to be included in search, default is False
:return list: list of broadcast items
"""
query = {
"query": {
"bool": {
"filter": [
{
"term": {
"genre.name": BROADCAST_GENRE
}
},
{
"terms": {
"broadcast.master_id": ids
}
},
],
}
}
}
req = ParsedRequest()
repos = "archive,published"
if include_archived_repo:
repos = "archive,published,archived"
req.args = {"source": json.dumps(query), "repo": repos}
return get_resource_service("search").get(req=req, lookup=None)
def test_compare_repos(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {}
req.max_results = 25
data = [{'headline': 'test {}'.format(i), 'slugline': 'rebuild {}'.format(i),
'type': 'text' if (i % 2 == 0) else 'picture'} for i in range(1, 100)]
resolve_document_etag(data, 'archive')
superdesk.app.data._search_backend('archive').bulk_insert('archive', data)
get_resource_service('archive').post(data)
consistency_init(self.app)
items = get_resource_service('archive').get(req, {})
self.assertEquals(99, items.count())
consistency_record = CompareRepositories().run('archive',
self.app.config['ELASTICSEARCH_URL'],
self.app.config['ELASTICSEARCH_INDEXES'].get('archive'))
self.assertEquals(consistency_record['mongo'], 99)
self.assertEquals(consistency_record['elastic'], 198)
self.assertEquals(consistency_record['identical'], 99)
self.assertEquals(consistency_record['mongo_only'], 0)
self.assertEquals(consistency_record['elastic_only'], 99)
self.assertEquals(consistency_record['inconsistent'], 0)
def get_elastic_item(self, id):
resource = superdesk.get_resource_service(self.resource_name)
query = {'query': {'filtered': {'filter': {'term': {'_id': id}}}}}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
items = resource.get(req=request, lookup=None)
return items[0]
def _get_broadcast_items(self, ids, include_archived_repo=False):
"""Returns list of broadcast items.
Get the broadcast items for the master_id
:param list ids: list of item ids
:param include_archived_repo True if archived repo needs to be included in search, default is False
:return list: list of broadcast items
"""
query = {
'query': {
'bool': {
'filter': [
{
'term': {
'genre.name': BROADCAST_GENRE
}
},
{
'terms': {
'broadcast.master_id': ids
}
},
],
}
}
}
req = ParsedRequest()
repos = 'archive,published'
if include_archived_repo:
repos = 'archive,published,archived'
req.args = {'source': json.dumps(query), 'repo': repos}
return get_resource_service('search').get(req=req, lookup=None)
def filter_subscribers(self, doc, subscribers, target_media_type):
"""
Filter subscribers to whom the current story is going to be delivered.
"""
filtered_subscribers = []
req = ParsedRequest()
req.args = {'is_global': True}
service = get_resource_service('publish_filters')
global_filters = list(service.get(req=req, lookup=None))
for subscriber in subscribers:
if target_media_type:
can_send_takes_packages = subscriber['subscriber_type'] == SUBSCRIBER_TYPES.DIGITAL
if target_media_type == WIRE and can_send_takes_packages or target_media_type == DIGITAL \
and not can_send_takes_packages:
continue
if doc.get('targeted_for'):
matching_target = [t for t in doc.get('targeted_for')
if t['name'] == subscriber.get('subscriber_type', '') or
t['name'] == subscriber.get('geo_restrictions', '')]
if len(matching_target) > 0 and matching_target[0]['allow'] is False:
continue
if not self.conforms_global_filter(subscriber, global_filters, doc):
continue
if not self.conforms_publish_filter(subscriber, doc):
continue
filtered_subscribers.append(subscriber)
return filtered_subscribers
def test_versions_across_collections_after_publish(self):
self.app.data.insert('archive_versions', self.article_versions)
# Publishing an Article
doc = self.articles[3]
original = doc.copy()
published_version_number = original[config.VERSION] + 1
get_resource_service(ARCHIVE_PUBLISH).patch(id=doc[config.ID_FIELD],
updates={ITEM_STATE: CONTENT_STATE.PUBLISHED,
config.VERSION: published_version_number})
article_in_production = get_resource_service(ARCHIVE).find_one(req=None, _id=original[config.ID_FIELD])
self.assertIsNotNone(article_in_production)
self.assertEqual(article_in_production[ITEM_STATE], CONTENT_STATE.PUBLISHED)
self.assertEqual(article_in_production[config.VERSION], published_version_number)
enqueue_published()
lookup = {'item_id': original[config.ID_FIELD], 'item_version': published_version_number}
queue_items = list(get_resource_service(PUBLISH_QUEUE).get(req=None, lookup=lookup))
assert len(queue_items) > 0, \
"Transmission Details are empty for published item %s" % original[config.ID_FIELD]
lookup = {'item_id': original[config.ID_FIELD], config.VERSION: published_version_number}
request = ParsedRequest()
request.args = {'aggregations': 0}
items_in_published_collection = list(get_resource_service(PUBLISHED).get(req=request, lookup=lookup))
assert len(items_in_published_collection) > 0, \
"Item not found in published collection %s" % original[config.ID_FIELD]
def get_rewritten_take_packages_per_event(self, event_id):
""" Returns all the published and rewritten take stories for the same event """
try:
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'package_type': 'takes'
}
}, {
'term': {
'event_id': event_id
}
}, {
'exists': {
'field': 'rewritten_by'
}
}]
}
}
}
}
}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
return super().get(req=request, lookup=None)
except:
return []
def test_versions_across_collections_after_publish(self):
self.app.data.insert('archive_versions', self.article_versions)
# Publishing an Article
doc = self.articles[7]
original = doc.copy()
published_version_number = original[config.VERSION] + 1
get_resource_service(ARCHIVE_PUBLISH).patch(id=doc[config.ID_FIELD],
updates={ITEM_STATE: CONTENT_STATE.PUBLISHED,
config.VERSION: published_version_number})
article_in_production = get_resource_service(ARCHIVE).find_one(req=None, _id=original[config.ID_FIELD])
self.assertIsNotNone(article_in_production)
self.assertEqual(article_in_production[ITEM_STATE], CONTENT_STATE.PUBLISHED)
self.assertEqual(article_in_production[config.VERSION], published_version_number)
enqueue_published()
lookup = {'item_id': original[config.ID_FIELD], 'item_version': published_version_number}
queue_items = list(get_resource_service(PUBLISH_QUEUE).get(req=None, lookup=lookup))
assert len(queue_items) > 0, \
"Transmission Details are empty for published item %s" % original[config.ID_FIELD]
lookup = {'item_id': original[config.ID_FIELD], config.VERSION: published_version_number}
request = ParsedRequest()
request.args = {'aggregations': 0}
items_in_published_collection = list(get_resource_service(PUBLISHED).get(req=request, lookup=lookup))
assert len(items_in_published_collection) > 0, \
"Item not found in published collection %s" % original[config.ID_FIELD]
def test_remove_by_id(self):
with self.app.app_context():
self.ids = self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}])
self.app.data.remove('items', {'_id': self.ids[0]})
req = ParsedRequest()
req.args = {}
self.assertEqual(1, self.app.data.find('items', req, None).count())
def test_using_repo_request_attribute(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {'repo': 'ingest'}
docs = self.app.data.find('search', req, None)[0]
self.assertEquals(1, docs.count())
self.assertEquals('ingest', docs[0]['_type'])
def get_product_items(self, product_id, size):
query = _items_query()
product = get_resource_service('products').find_one(req=None,
_id=product_id)
if not product:
return
query['bool']['should'] = []
get_resource_service('section_filters').apply_section_filter(
query, product.get('product_type'))
if product.get('sd_product_id'):
query['bool']['should'].append(
{'term': {
'products.code': product['sd_product_id']
}})
if product.get('query'):
query['bool']['should'].append(query_string(product['query']))
query['bool']['minimum_should_match'] = 1
source = {'query': query}
source['sort'] = [{'versioncreated': 'desc'}]
source['size'] = size
source['from'] = 0
source['post_filter'] = {'bool': {'must': []}}
internal_req = ParsedRequest()
internal_req.args = {'source': json.dumps(source)}
return list(super().get(internal_req, None))
def _get_combined_view_data(self, items, request):
"""Get list of event and planning for the combined view
:param items:
:param request: object representing the HTTP request
"""
ids = set()
for item in items:
# don't want related planing items
_id = item.get('event_item') or item.get('_id')
ids.add(_id)
filters = self._get_date_filters(request)
page = request.page or 1
page_size = self._get_page_size(request)
req = ParsedRequest()
req.args = MultiDict()
req.args['source'] = json.dumps({
'query': {
'bool': {
'must': [{'terms': {'_id': list(ids)}}],
}
},
'filter': filters,
'sort': self._get_sort(),
'size': self._get_page_size(request),
'from': (page - 1) * page_size
})
req.page = request.page or 1
req.max_results = self._get_page_size(request)
return get_resource_service('planning_search').get(req=req, lookup=None)
def test_using_repo_request_attribute(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {'repo': 'ingest'}
docs = self.app.data.find('search', req, None)
self.assertEquals(1, docs.count())
self.assertEquals('ingest', docs[0]['_type'])
def _get_changed_items(self, existing_items, package):
"""Returns the added and removed items from existing_items
:param existing_items: Existing list
:param updates: Changes
:return: list of removed items and list of added items
"""
published_service = get_resource_service('published')
req = ParsedRequest()
query = {'query': {'filtered': {'filter': {'and': [{'terms': {QUEUE_STATE: [
PUBLISH_STATE.QUEUED, PUBLISH_STATE.QUEUED_NOT_TRANSMITTED]}},
{'term': {'item_id': package['item_id']}}]}}}, 'sort': [{'publish_sequence_no': 'desc'}]}
req.args = {'source': json.dumps(query)}
req.max_results = 1
previously_published_packages = published_service.get(req=req, lookup=None)
if not previously_published_packages.count():
return [], []
previously_published_package = previously_published_packages[0]
if 'groups' in previously_published_package:
old_items = self.package_service.get_residrefs(previously_published_package)
added_items = list(set(existing_items) - set(old_items))
removed_items = list(set(old_items) - set(existing_items))
return removed_items, added_items
else:
return [], []
def test_compare_repos(self):
with self.app.app_context():
req = ParsedRequest()
req.args = {}
req.max_results = 25
data = [{
'headline': 'test {}'.format(i),
'slugline': 'rebuild {}'.format(i),
'type': 'text' if (i % 2 == 0) else 'picture'
} for i in range(1, 100)]
resolve_document_etag(data, 'archive')
superdesk.app.data._search_backend('archive').bulk_insert(
'archive', data)
get_resource_service('archive').post(data)
consistency_init(self.app)
items = get_resource_service('archive').get(req, {})
self.assertEquals(99, items.count())
consistency_record = CompareRepositories().run(
'archive', self.app.config['ELASTICSEARCH_URL'],
self.app.config['ELASTICSEARCH_INDEXES'].get('archive'))
self.assertEquals(consistency_record['mongo'], 99)
self.assertEquals(consistency_record['elastic'], 198)
self.assertEquals(consistency_record['identical'], 99)
self.assertEquals(consistency_record['mongo_only'], 0)
self.assertEquals(consistency_record['elastic_only'], 99)
self.assertEquals(consistency_record['inconsistent'], 0)
def get_items(self, item_ids):
try:
query = {
'bool': {
'must_not': [
{
'term': {
'type': 'composite'
}
},
],
'must': [{
'terms': {
'_id': item_ids
}
}],
}
}
source = {'query': query}
source['size'] = len(item_ids)
req = ParsedRequest()
req.args = {'source': json.dumps(source)}
return super().get(req, None)
except Exception as exc:
logger.error('Error in get_items for query: {}'.format(
json.dumps(source)),
exc,
exc_info=True)
def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal):
"""Get the pulished items where flag is moved.
:param list item_ids: List of item
:param bool move_to_legal: move_to_legal boolean flag
:return: list of published items
"""
if item_ids:
try:
query = {
'query': {
'filtered': {
'filter': {
'and': [
{'terms': {'item_id': item_ids}},
{'term': {'moved_to_legal': move_to_legal}}
]
}
}
}
}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
return list(super().get(req=request, lookup=None))
except:
logger.exception('Failed to get published items '
'by moved to legal: {} -- ids: {}.'.format(move_to_legal, item_ids))
return []
def get_rewritten_items_by_event_story(self, event_id, rewrite_id):
""" Returns all the published and rewritten stories for the given event and rewrite_id"""
try:
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': [{
'term': {
'event_id': event_id
}
}, {
'term': {
'rewritten_by': rewrite_id
}
}]
}
}
}
}
}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
return super().get(req=request, lookup=None)
except:
return []
def _get_broadcast_items(self, ids):
"""
Get the broadcast items for the master_id and takes_package_id
:param list ids: list of item ids
:return list: list of broadcast items
"""
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': {'term': {'genre.name': BROADCAST_GENRE}},
'should': [
{'terms': {'broadcast.master_id': ids}},
{'terms': {'broadcast.takes_package_id': ids}}
]
}
}
}
}
}
req = ParsedRequest()
req.args = {'source': json.dumps(query), 'repo': 'archive,published'}
return get_resource_service('search').get(req=req, lookup=None)
def test_search_capi_aggregations(self):
self.content_api.publish(
{
'_id': '1',
'guid': '1',
'type': 'text',
'anpa_category': [{
'qcode': 'i',
'name': 'International News'
}],
'headline': 'Man bites dog',
'source': 'AAA',
'urgency': 1
}, [])
self.content_api.publish(
{
'_id': '2',
'guid': '2',
'type': 'text',
'anpa_category': [{
'qcode': 'i',
'name': 'International News'
}],
'headline': 'Man bites cat',
'source': 'BBB',
'urgency': 2
}, [])
test = superdesk.get_resource_service('search_capi')
req = ParsedRequest()
req.args = MultiDict([('aggregations', 1)])
resp = test.get(req=req, lookup=None)
self.assertEqual(
resp.hits['aggregations']['category']['buckets'][0]['doc_count'],
2)
def assert_products_query(user_id, args=None, products=None):
with app.test_request_context():
session['user'] = user_id
search = SearchQuery()
if args is None:
service.prefill_search_query(search)
else:
req = ParsedRequest()
req.args = args
service.prefill_search_query(search, req)
service.apply_products_filter(search)
for product in products:
if product.get('query'):
assert {
'query_string': {
'query': product['query'],
'default_operator': 'AND',
'lenient': True
}
} in search.query['bool']['should']
sd_product_ids = [
product['sd_product_id'] for product in products
if product.get('sd_product_id')
]
if len(sd_product_ids):
assert {
'terms': {
'products.code': sd_product_ids
}
} in search.query['bool']['should']
def get_last_published_version(self, _id):
"""Returns the last published entry for the passed item id
:param _id:
:return:
"""
try:
query = {
"query": {
"filtered": {
"filter": {
"bool": {
"must": [{
"term": {
"item_id": _id
}
}, {
"term": {
LAST_PUBLISHED_VERSION: True
}
}]
}
}
}
}
}
request = ParsedRequest()
request.args = {"source": json.dumps(query), "repo": "published"}
items = list(self.get(req=request, lookup=None))
if items:
return items[0]
except Exception:
return None
def get_published_items_by_moved_to_legal(self, item_ids, move_to_legal):
"""Get the pulished items where flag is moved.
:param list item_ids: List of item
:param bool move_to_legal: move_to_legal boolean flag
:return: list of published items
"""
if item_ids:
try:
query = {
"query": {
"filtered": {
"filter": {
"and": [{"terms": {"item_id": item_ids}}, {"term": {"moved_to_legal": move_to_legal}}]
}
}
}
}
request = ParsedRequest()
request.args = {"source": json.dumps(query)}
return list(super().get(req=request, lookup=None))
except:
logger.exception(
"Failed to get published items " "by moved to legal: {} -- ids: {}.".format(move_to_legal, item_ids)
)
return []
def _enchance_assignment(self, doc):
"""Populate `item_ids` with ids for all linked Archive items for an Assignment
Using the `search` resource service, retrieve the list of Archive items linked to
the provided Assignment.
"""
query = {
'query': {
'filtered': {
'filter': {
'bool': {
'must': {
'term': {
'assignment_id': str(doc[config.ID_FIELD])
}
},
}
}
}
}
}
req = ParsedRequest()
repos = 'archive,published,archived'
req.args = {'source': json.dumps(query), 'repo': repos}
items = list(get_resource_service('search').get(req=req, lookup=None))
if items:
doc['item_ids'] = [
str(item.get(config.ID_FIELD)) for item in items
]
self.set_type(doc, doc)
def test_remove_with_query(self):
with self.app.app_context():
self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}])
self.app.data.remove('items', {'query': {'term': {'uri': 'bar'}}})
req = ParsedRequest()
req.args = {}
self.assertEqual(1, self.app.data.find('items', req, None).count())
def test_args_filter(self):
with self.app.app_context():
self.app.data.insert('items', [{'uri': 'foo'}, {'uri': 'bar'}])
req = ParsedRequest()
req.args = {}
req.args['filter'] = json.dumps({'term': {'uri': 'foo'}})
self.assertEqual(1, self.app.data.find('items', req, None).count())
def test_resource_filter(self):
with self.app.app_context():
self.app.data.insert('items_with_description', [{'uri': 'foo', 'description': 'test'}, {'uri': 'bar'}])
req = ParsedRequest()
req.args = {}
req.args['source'] = json.dumps({'query': {'filtered': {'filter': {'term': {'uri': 'bar'}}}}})
self.assertEqual(0, self.app.data.find('items_with_description', req, None).count())
def test_sub_resource_lookup_with_schema_filter(self):
with self.app.app_context():
self.app.data.insert('items_with_description', [{'uri': 'foo', 'description': 'test', 'name': 'foo'}])
req = ParsedRequest()
req.args = {}
self.assertEqual(1, self.app.data.find('items_with_description', req, {'name': 'foo'}).count())
self.assertEqual(0, self.app.data.find('items_with_description', req, {'name': 'bar'}).count())
def test_sub_resource_lookup(self):
with self.app.app_context():
self.app.data.insert('items', [{'uri': 'foo', 'name': 'foo'}])
req = ParsedRequest()
req.args = {}
self.assertEqual(1, self.app.data.find('items', req, {'name': 'foo'}).count())
self.assertEqual(0, self.app.data.find('items', req, {'name': 'bar'}).count())
def test_search_capi_filter(self):
subscriber = {'_id': 'sub1'}
self.content_api.publish({'_id': 'foo', 'guid': 'foo', 'type': 'text',
'anpa_category': [{'qcode': 'i', 'name': 'International News'}],
'headline': 'Man bites dog'}, [subscriber])
self.content_api.publish({'_id': 'bar', 'guid': 'bar',
'anpa_category': [{'qcode': 'i', 'name': 'International News'}],
'type': 'text'}, [{'_id': 'sub2'}])
self.content_api.publish({'_id': 'nat', 'guid': 'nat',
'anpa_category': [{'qcode': 'a', 'name': 'National News'}],
'type': 'text'}, [{'_id': 'sub2'}])
test = superdesk.get_resource_service('search_capi')
req = ParsedRequest()
req.args = MultiDict([('filter', '[{"term": {"service.code": "i"}}]')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 2)
self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'i')
self.assertEqual(resp.docs[1].get('anpa_category')[0].get('qcode'), 'i')
req = ParsedRequest()
req.args = MultiDict([('service', 'i')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 2)
self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'i')
self.assertEqual(resp.docs[1].get('anpa_category')[0].get('qcode'), 'i')
req = ParsedRequest()
req.args = MultiDict([('service', '["a"]')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'a')
req = ParsedRequest()
req.args = MultiDict([('service', 'i'), ('subscribers', 'sub1')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
self.assertEqual(resp.docs[0].get('anpa_category')[0].get('qcode'), 'i')
self.assertEqual(resp.docs[0].get('subscribers')[0], 'sub1')
req = ParsedRequest()
req.args = MultiDict([('subscribers', 'sub2')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 2)
self.assertEqual(resp.docs[0].get('subscribers')[0], 'sub2')
self.assertEqual(resp.docs[1].get('subscribers')[0], 'sub2')
def get_other_published_items(self, _id):
try:
query = {"query": {"filtered": {"filter": {"term": {"item_id": _id}}}}}
request = ParsedRequest()
request.args = {"source": json.dumps(query)}
return super().get(req=request, lookup=None)
except:
return []
def get_other_published_items(self, _id):
try:
query = {'query': {'filtered': {'filter': {'term': {'item_id': _id}}}}}
request = ParsedRequest()
request.args = {'source': json.dumps(query)}
return super().get(req=request, lookup=None)
except:
return []
