def _get_field_values(self):
values = {}
vocabularies_resource = get_resource_service('vocabularies')
values['anpa_category'] = vocabularies_resource.find_one(req=None, _id='categories')['items']
req = ParsedRequest()
req.where = json.dumps({'$or': [{"schema_field": "genre"}, {"_id": "genre"}]})
genre = vocabularies_resource.get(req=req, lookup=None)
if genre.count():
values['genre'] = genre[0]['items']
values['urgency'] = vocabularies_resource.find_one(req=None, _id='urgency')['items']
values['priority'] = vocabularies_resource.find_one(req=None, _id='priority')['items']
values['type'] = vocabularies_resource.find_one(req=None, _id='type')['items']
subject = vocabularies_resource.find_one(req=None, schema_field='subject')
if subject:
values['subject'] = subject['items']
else:
values['subject'] = get_subjectcodeitems()
values['desk'] = list(get_resource_service('desks').get(None, {}))
values['stage'] = self._get_stage_field_values(values['desk'])
values['sms'] = [{'qcode': 0, 'name': 'False'}, {'qcode': 1, 'name': 'True'}]
values['embargo'] = [{'qcode': 0, 'name': 'False'}, {'qcode': 1, 'name': 'True'}]
req = ParsedRequest()
req.where = json.dumps({'$or': [{"schema_field": "place"}, {"_id": "place"}, {"_id": "locators"}]})
place = vocabularies_resource.get(req=req, lookup=None)
if place.count():
values['place'] = place[0]['items']
values['ingest_provider'] = list(get_resource_service('ingest_providers').get(None, {}))
return values
def get(self, req, lookup):
"""Filter out personal activity on personal items if inquired by another user."""
if req is None:
req = ParsedRequest()
user = getattr(g, 'user', None)
if not user:
raise SuperdeskApiError.notFoundError('Can not determine user')
where_cond = {}
if req.where:
if req.where[0] != '{':
req.where = '{' + req.where + '}'
where_cond = json.loads(req.where)
for_user = where_cond.get('user', str(user.get('_id')))
if for_user != str(user.get('_id')):
where_item = {
'$and': [{
'desk': {
'$ne': None
}
}, {
'desk': {
'$exists': True
}
}, {
'resource': 'archive'
}]
}
where_cond['$or'] = [where_item, {'resource': {'$ne': 'archive'}}]
req.where = json.dumps(where_cond)
return self.backend.get(self.datasource, req=req, lookup=lookup)
def get(self, req, lookup):
"""Filter out personal activity on personal items if inquired by another user."""
if req is None:
req = ParsedRequest()
user = getattr(g, "user", None)
if not user:
raise SuperdeskApiError.notFoundError("Can not determine user")
where_cond = {}
if req.where:
if req.where[0] != "{":
req.where = "{" + req.where + "}"
where_cond = json.loads(req.where)
for_user = where_cond.get("user", str(user.get("_id")))
if for_user != str(user.get("_id")):
where_item = {
"$and": [{
"desk": {
"$ne": None
}
}, {
"desk": {
"$exists": True
}
}, {
"resource": "archive"
}]
}
where_cond["$or"] = [where_item, {"resource": {"$ne": "archive"}}]
req.where = json.dumps(where_cond)
return self.backend.get(self.datasource, req=req, lookup=lookup)
def _get_field_values(self):
values = {}
vocabularies_resource = get_resource_service("vocabularies")
categories_cv = vocabularies_resource.find_one(req=None, _id="categories")
values["anpa_category"] = categories_cv.get("items") if categories_cv else []
req = ParsedRequest()
req.where = json.dumps({"$or": [{"schema_field": "genre"}, {"_id": "genre"}]})
genre = vocabularies_resource.get(req=req, lookup=None)
if genre.count():
values["genre"] = genre[0]["items"]
for voc_id in ("urgency", "priority", "type"):
try:
values[voc_id] = vocabularies_resource.find_one(req=None, _id=voc_id)["items"]
except TypeError:
values[voc_id] = []
subject = vocabularies_resource.find_one(req=None, schema_field="subject")
if subject:
values["subject"] = subject["items"]
else:
values["subject"] = get_subjectcodeitems()
values["desk"] = list(get_resource_service("desks").get(None, {}))
values["stage"] = self._get_stage_field_values(values["desk"])
values["sms"] = [{"qcode": 0, "name": "False"}, {"qcode": 1, "name": "True"}]
values["embargo"] = [{"qcode": 0, "name": "False"}, {"qcode": 1, "name": "True"}]
req = ParsedRequest()
req.where = json.dumps({"$or": [{"schema_field": "place"}, {"_id": "place"}, {"_id": "locators"}]})
place = vocabularies_resource.get(req=req, lookup=None)
if place.count():
values["place"] = place[0]["items"]
else:
values["place"] = []
values["ingest_provider"] = list(get_resource_service("ingest_providers").get(None, {}))
values["featuremedia"] = [{"qcode": 1, "name": "True"}, {"qcode": 0, "name": "False"}]
return values
def _get_field_values(self):
values = {}
vocabularies_resource = get_resource_service('vocabularies')
values['anpa_category'] = vocabularies_resource.find_one(req=None, _id='categories')['items']
req = ParsedRequest()
req.where = json.dumps({'$or': [{"schema_field": "genre"}, {"_id": "genre"}]})
genre = vocabularies_resource.get(req=req, lookup=None)
if genre.count():
values['genre'] = genre[0]['items']
values['urgency'] = vocabularies_resource.find_one(req=None, _id='urgency')['items']
values['priority'] = vocabularies_resource.find_one(req=None, _id='priority')['items']
values['type'] = vocabularies_resource.find_one(req=None, _id='type')['items']
subject = vocabularies_resource.find_one(req=None, schema_field='subject')
if subject:
values['subject'] = subject['items']
else:
values['subject'] = get_subjectcodeitems()
values['desk'] = list(get_resource_service('desks').get(None, {}))
values['stage'] = self._get_stage_field_values(values['desk'])
values['sms'] = [{'qcode': 0, 'name': 'False'}, {'qcode': 1, 'name': 'True'}]
values['embargo'] = [{'qcode': 0, 'name': 'False'}, {'qcode': 1, 'name': 'True'}]
req = ParsedRequest()
req.where = json.dumps({'$or': [{"schema_field": "place"}, {"_id": "place"}, {"_id": "locators"}]})
place = vocabularies_resource.get(req=req, lookup=None)
if place.count():
values['place'] = place[0]['items']
values['ingest_provider'] = list(get_resource_service('ingest_providers').get(None, {}))
return values
def get_items(self, now):
"""Get the items from the archive collection that have expiry in future
and state is published, corrected, killed
:param datetime now: current date time
:return list: list of expired items
"""
logger.info('Fetching expired items from archive collection.')
now = now + timedelta(minutes=self.expiry_minutes)
query = {
'expiry': {
'$gte': date_to_str(now)
},
ITEM_STATE: {
'$in': [
CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED,
CONTENT_STATE.KILLED, CONTENT_STATE.RECALLED
]
}
}
req = ParsedRequest()
req.sort = '[("unique_id", 1)]'
req.where = json.dumps(query)
cursor = get_resource_service(ARCHIVE).get_from_mongo(req=req,
lookup=None)
count = cursor.count()
no_of_pages = 0
if count:
no_of_pages = len(range(0, count, self.default_page_size))
unique_id = cursor[0]['unique_id']
logger.info('Number of items to modify: {}, pages={}'.format(
count, no_of_pages))
else:
logger.info('No items to modify.')
for page in range(0, no_of_pages):
logger.info(
'Fetching items for page number: {} unique_id: {}'.format(
(page + 1), unique_id))
req = ParsedRequest()
req.sort = '[("unique_id", 1)]'
if page == 0:
query['unique_id'] = {'$gte': unique_id}
else:
query['unique_id'] = {'$gt': unique_id}
req.where = json.dumps(query)
req.max_results = self.default_page_size
cursor = get_resource_service(ARCHIVE).get_from_mongo(req=req,
lookup=None)
items = list(cursor)
if len(items) > 0:
unique_id = items[len(items) - 1]['unique_id']
logger.info('Fetched No. of Items: {} for page: {}'.format(
len(items), (page + 1)))
yield items
def get_publish_queue_items(self, page_size, expired_items=None):
"""Get publish queue items that are not moved to legal
:param int page_size: batch size
:param list expired_items:
:return list: publish queue items
"""
if expired_items is None:
expired_items = []
query = {"moved_to_legal": False}
if expired_items:
query["item_id"] = {"$in": expired_items}
else:
query["state"] = {
"$in": [
QueueState.SUCCESS.value, QueueState.CANCELED.value,
QueueState.FAILED.value
]
}
service = get_resource_service("publish_queue")
req = ParsedRequest()
req.sort = '[("_id", 1)]'
req.where = json.dumps(query)
cursor = service.get(req=req, lookup=None)
count = cursor.count()
no_of_pages = 0
if count:
no_of_pages = len(range(0, count, page_size))
queue_id = cursor[0][config.ID_FIELD]
logger.info(
"Number of items to move to legal archive publish queue: {}, pages={}"
.format(count, no_of_pages))
for page in range(0, no_of_pages):
logger.info("Fetching publish queue items "
"for page number: {}. queue_id: {}".format((page + 1),
queue_id))
req = ParsedRequest()
req.sort = '[("_id", 1)]'
query["_id"] = {"$gte": str(queue_id)}
req.where = json.dumps(query)
req.max_results = page_size
cursor = service.get(req=req, lookup=None)
items = list(cursor)
if len(items) > 0:
queue_id = items[len(items) - 1][config.ID_FIELD]
logger.info("Fetched No. of Items: {} for page: {} "
"For import in to legal archive publish_queue.".format(
len(items), (page + 1)))
yield items
def get_items(self, now):
"""Get the items from the archive collection that have expiry in future
and state is published, corrected, killed
:param datetime now: current date time
:return list: list of expired items
"""
logger.info('Fetching expired items from archive collection.')
now = now + timedelta(minutes=self.expiry_minutes)
query = {
'expiry': {'$gte': date_to_str(now)},
ITEM_STATE: {'$in': [
CONTENT_STATE.PUBLISHED,
CONTENT_STATE.CORRECTED,
CONTENT_STATE.KILLED,
CONTENT_STATE.RECALLED
]}
}
req = ParsedRequest()
req.sort = '[("unique_id", 1)]'
req.where = json.dumps(query)
cursor = get_resource_service(ARCHIVE).get_from_mongo(req=req, lookup=None)
count = cursor.count()
no_of_pages = 0
if count:
no_of_pages = len(range(0, count, self.default_page_size))
unique_id = cursor[0]['unique_id']
logger.info('Number of items to modify: {}, pages={}'.format(count, no_of_pages))
else:
logger.info('No items to modify.')
for page in range(0, no_of_pages):
logger.info('Fetching items for page number: {} unique_id: {}'. format((page + 1), unique_id))
req = ParsedRequest()
req.sort = '[("unique_id", 1)]'
if page == 0:
query['unique_id'] = {'$gte': unique_id}
else:
query['unique_id'] = {'$gt': unique_id}
req.where = json.dumps(query)
req.max_results = self.default_page_size
cursor = get_resource_service(ARCHIVE).get_from_mongo(req=req, lookup=None)
items = list(cursor)
if len(items) > 0:
unique_id = items[len(items) - 1]['unique_id']
logger.info('Fetched No. of Items: {} for page: {}'.format(len(items), (page + 1)))
yield items
def get(self, req, lookup):
"""
Overriding to pass user as a search parameter
"""
session_user = str(get_user_id(required=True))
if not req:
req = ParsedRequest()
if lookup:
req.where = json.dumps({'$or': [{'is_global': True}, {'user': session_user}, lookup]})
else:
req.where = json.dumps({'$or': [{'is_global': True}, {'user': session_user}]})
return super().get(req, lookup=None)
def _get_items(self):
"""Get items from the LegalArchive that belong to the COMMISSION desk
:return: list: list of legal archive content
"""
logger.info('Fetching legal archive content for the {} desk'.format(
self.default_desk))
query = {'task.desk': self.default_desk, 'type': 'text'}
req = ParsedRequest()
req.where = json.dumps(query)
legal_archive_service = get_resource_service(LEGAL_ARCHIVE_NAME)
legal_items = list(
legal_archive_service.get_from_mongo(req=req, lookup=None))
if legal_items:
logger.info(
'Found {} items in the legal archive for the {} desk'.format(
len(legal_items), self.default_desk))
else:
logger.warning(
'Failed to find any {} desk items in the legal archive'.format(
self.default_desk))
legal_items = []
return legal_items
def get(self, req, lookup):
"""Retrieve a list of items that match the filter criteria (if any)
pssed along the HTTP request.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict lookup: sub-resource lookup from the endpoint URL
:return: database results cursor object
:rtype: `pymongo.cursor.Cursor`
"""
if req is None:
req = ParsedRequest()
request_params = req.args or {}
allowed_params = ('q',)
for param in request_params.keys():
if param not in allowed_params:
raise UnexpectedParameterError(
desc="Unexpected parameter ({})".format(param)
)
if 'q' in request_params:
req.where = request_params['q']
return super().get(req, lookup)
def run(self, page_size=None):
logger.info("Import to Legal Archive")
lock_name = get_lock_id("legal_archive", "import_to_legal_archive")
page_size = int(page_size) if page_size else self.default_page_size
if not lock(lock_name, "", expire=1800):
return
try:
legal_archive_import = LegalArchiveImport()
publish_queue = get_resource_service("publish_queue")
for items in self.get_expired_items(page_size):
for item in items:
try:
legal_archive_import.upsert_into_legal_archive(item.get("item_id"))
req = ParsedRequest()
req.where = json.dumps({"item_id": item["item_id"]})
queue_items = list(publish_queue.get(req=req, lookup=None))
if queue_items:
try:
logger.info("Import to Legal Publish Queue")
legal_archive_import.process_queue_items(queue_items, True)
except:
logger.exception(
"Failed to import into legal publish queue "
"archive via command {}.".format(item.get("item_id"))
)
except:
logger.exception(
"Failed to import into legal " "archive via command {}.".format(item.get("item_id"))
)
except:
logger.exception("Failed to import into legal archive.")
finally:
unlock(lock_name, "")
def _get_items(self):
"""Get items from the LegalArchive that belong to the COMMISSION desk
:return: list: list of legal archive content
"""
logger.info('Fetching legal archive content for the {} desk'.format(self.default_desk))
query = {'task.desk': self.default_desk, 'type': 'text'}
req = ParsedRequest()
req.where = json.dumps(query)
legal_archive_service = get_resource_service(LEGAL_ARCHIVE_NAME)
legal_items = list(legal_archive_service.get_from_mongo(req=req, lookup=None))
if legal_items:
logger.info(
'Found {} items in the legal archive for the {} desk'.format(
len(legal_items),
self.default_desk)
)
else:
logger.warning('Failed to find any {} desk items in the legal archive'.format(self.default_desk))
legal_items = []
return legal_items
def get_history_items(self, last_id, gte, item_id, chunk_size=0):
history_service = get_resource_service('archive_history')
last_processed_id = last_id
while True:
req = ParsedRequest()
req.sort = '[("_id", 1), ("version", 1)]'
query = {'$and': []}
if gte:
query['$and'].append({'_created': {'$gte': date_to_str(gte)}})
if item_id:
query['$and'].append({'item_id': str(item_id)})
if last_processed_id:
query['$and'].append({'_id': {'$gt': str(last_processed_id)}})
req.where = json.dumps(query)
if chunk_size > 0:
req.max_results = int(chunk_size)
items = list(history_service.get(req=req, lookup=None))
if len(items) < 1:
break
last_processed_id = items[-1][config.ID_FIELD]
yield items
def update_media_references(self, updates, original, published=False):
"""Update the media references collection.
When item (media item or associated media) is updated or created,
media_references are created. These media_references are updated to published state
once the item is published.
:param dict updates: Updates of the item
:param dict original: Original item
:param boolean published: True if publishing the item else False
"""
item_id = original.get(config.ID_FIELD)
references = {}
if updates.get('renditions', original.get('renditions', {})):
references = {
item_id: updates.get('renditions', original.get('renditions', {}))
}
if original.get(ITEM_TYPE) not in MEDIA_TYPES:
associations = updates.get(ASSOCIATIONS) or original.get(ASSOCIATIONS)
if not associations:
return
references = {assoc.get(config.ID_FIELD): assoc.get('renditions')
for assoc in associations.values() if assoc and assoc.get('renditions')}
if not references:
return
for assoc_id, renditions in references.items():
associated_id = assoc_id if assoc_id != item_id else None
for rendition in renditions.values():
if not rendition.get('media'):
continue
media = str(rendition.get('media'))
reference = get_resource_service('media_references').find_one(req=None, item_id=item_id,
media_id=media)
if not reference:
try:
get_resource_service('media_references').post([{'item_id': item_id,
'media_id': media,
'associated_id': associated_id,
'published': False}])
except:
logger.exception('Failed to insert media reference item {} media {}'.format(item_id, media))
# item is publish
if not published:
return
req = ParsedRequest()
req.where = json.dumps({'item_id': item_id, 'published': False})
refs = list(get_resource_service('media_references').get(req=req, lookup=None))
for ref in refs:
try:
get_resource_service('media_references').patch(ref.get(config.ID_FIELD),
updates={'published': True})
except:
logger.exception('Failed to update media '
'reference item {} media {}'.format(ref.get("item_id"), ref.get("media_id")))
def update_media_references(self, updates, original, published=False):
"""Update the media references collection.
When item (media item or associated media) is updated or created,
media_references are created. These media_references are updated to published state
once the item is published.
:param dict updates: Updates of the item
:param dict original: Original item
:param boolean published: True if publishing the item else False
"""
item_id = original.get(config.ID_FIELD)
references = {}
if updates.get('renditions', original.get('renditions', {})):
references = {
item_id: updates.get('renditions', original.get('renditions', {}))
}
if original.get(ITEM_TYPE) not in MEDIA_TYPES:
associations = updates.get(ASSOCIATIONS) or original.get(ASSOCIATIONS)
if not associations:
return
references = {assoc.get(config.ID_FIELD): assoc.get('renditions')
for assoc in associations.values() if assoc and assoc.get('renditions')}
if not references:
return
for assoc_id, renditions in references.items():
associated_id = assoc_id if assoc_id != item_id else None
for rendition in renditions.values():
if not rendition.get('media'):
continue
media = str(rendition.get('media'))
reference = get_resource_service('media_references').find_one(req=None, item_id=item_id,
media_id=media)
if not reference:
try:
get_resource_service('media_references').post([{'item_id': item_id,
'media_id': media,
'associated_id': associated_id,
'published': False}])
except Exception:
logger.exception('Failed to insert media reference item {} media {}'.format(item_id, media))
# item is publish
if not published:
return
req = ParsedRequest()
req.where = json.dumps({'item_id': item_id, 'published': False})
refs = list(get_resource_service('media_references').get(req=req, lookup=None))
for ref in refs:
try:
get_resource_service('media_references').patch(ref.get(config.ID_FIELD),
updates={'published': True})
except Exception:
logger.exception('Failed to update media '
'reference item {} media {}'.format(ref.get("item_id"), ref.get("media_id")))
def test_search_capi(self):
subscriber = {"_id": "sub1"}
self.content_api.publish(
{
"_id": "foo",
"guid": "foo",
"type": "text",
"anpa_category": [{"qcode": "i", "name": "International News"}],
"headline": "Man bites dog",
},
[subscriber],
)
self.content_api.publish({"_id": "bar", "guid": "bar", "type": "text"}, [{"_id": "sub2"}])
test = superdesk.get_resource_service("search_capi")
req = ParsedRequest()
req.args = MultiDict([("subscribers", "sub1")])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
resp = test.get(req=None, lookup=None)
self.assertEqual(resp.count(), 2)
req = ParsedRequest()
req.args = MultiDict()
req.where = '{"headline":"dog"}'
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
def get(self, req, lookup):
"""
Overriding because of a different resource URL and user_id is part of the URL
"""
req = ParsedRequest()
req.where = json.dumps(lookup)
return super().get(req, lookup=None)
def get(self, req, lookup):
"""
Overriding because of a different resource URL and user_id is part of the URL
"""
req = ParsedRequest()
req.where = json.dumps(lookup)
return super().get(req, lookup=None)
def test_find_where(data_layer):
resource = 'musicians'
sub_resource_lookup = None
req = ParsedRequest()
req.where = 'name == "Bill Evans"'
req.max_results = 1
results = data_layer.find(resource, req, sub_resource_lookup)
assert len(results) == 1
assert results[0]['name'] == 'Bill Evans'
def get(self, req, lookup):
"""
Overriding because of a different resource URL and user_id is part of the URL
"""
if not req:
req = ParsedRequest()
req.where = json.dumps({'$or': [lookup, {'is_global': True}]})
return super().get(req, lookup=None)
def get(self, req, lookup):
"""
Overriding because of a different resource URL and user_id is part of the URL
"""
if not req:
req = ParsedRequest()
req.where = json.dumps({'$or': [lookup, {'is_global': True}]})
return super().get(req, lookup=None)
def get_publish_queue_items(self, page_size, expired_items=[]):
"""Get publish queue items that are not moved to legal
:param int page_size: batch size
:param list expired_items:
:return list: publish queue items
"""
query = {"moved_to_legal": False}
if expired_items:
query["item_id"] = {"$in": expired_items}
else:
query["state"] = {"$in": [QueueState.SUCCESS.value, QueueState.CANCELED.value, QueueState.FAILED.value]}
service = get_resource_service("publish_queue")
req = ParsedRequest()
req.sort = '[("_id", 1)]'
req.where = json.dumps(query)
cursor = service.get(req=req, lookup=None)
count = cursor.count()
no_of_pages = 0
if count:
no_of_pages = len(range(0, count, page_size))
queue_id = cursor[0][config.ID_FIELD]
logger.info("Number of items to move to legal archive publish queue: {}, pages={}".format(count, no_of_pages))
for page in range(0, no_of_pages):
logger.info(
"Fetching publish queue items " "for page number: {}. queue_id: {}".format((page + 1), queue_id)
)
req = ParsedRequest()
req.sort = '[("_id", 1)]'
query["_id"] = {"$gte": str(queue_id)}
req.where = json.dumps(query)
req.max_results = page_size
cursor = service.get(req=req, lookup=None)
items = list(cursor)
if len(items) > 0:
queue_id = items[len(items) - 1][config.ID_FIELD]
logger.info(
"Fetched No. of Items: {} for page: {} "
"For import in to legal archive publish_queue.".format(len(items), (page + 1))
)
yield items
def test_import_into_legal_archive(self):
archive_publish = get_resource_service('archive_publish')
archive_correct = get_resource_service('archive_correct')
legal_archive = get_resource_service('legal_archive')
archive = get_resource_service('archive_publish')
published = get_resource_service('published')
publish_queue = get_resource_service('publish_queue')
self.original_method = LegalArchiveImport.upsert_into_legal_archive
LegalArchiveImport.upsert_into_legal_archive = MagicMock()
for item in self.archive_items:
archive_publish.patch(item['_id'], {'headline': 'publishing', 'abstract': 'publishing'})
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item['_id'])
self.assertIsNone(legal_item, 'Item: {} is not none.'.format(item['_id']))
archive_correct.patch(self.archive_items[1]['_id'], {'headline': 'correcting', 'abstract': 'correcting'})
LegalArchiveImport.upsert_into_legal_archive = self.original_method
self.class_under_test().run(1)
# items are not expired
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item['_id'])
self.assertIsNone(legal_item, 'Item: {} is not none.'.format(item['_id']))
# expire the items
for item in self.archive_items:
original = archive.find_one(req=None, _id=item['_id'])
archive.system_update(item['_id'], {'expiry': utcnow() - timedelta(minutes=30)}, original)
published.update_published_items(item['_id'], 'expiry', utcnow() - timedelta(minutes=30))
# run the command after expiry
self.class_under_test().run(1)
# items are expired
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item['_id'])
self.assertEqual(legal_item['_id'], item['_id'], 'item {} not imported to legal'.format(item['_id']))
# items are moved to legal
for item in self.archive_items:
published_items = list(published.get_other_published_items(item['_id']))
for published_item in published_items:
self.assertEqual(published_item['moved_to_legal'], True)
# items are moved to legal publish queue
for item in self.archive_items:
req = ParsedRequest()
req.where = json.dumps({'item_id': item['_id']})
queue_items = list(publish_queue.get(req=req, lookup=None))
self.assertGreaterEqual(len(queue_items), 1)
for queue_item in queue_items:
self.assertEqual(queue_item['moved_to_legal'], True)
def test_import_into_legal_archive(self):
archive_publish = get_resource_service('archive_publish')
archive_correct = get_resource_service('archive_correct')
legal_archive = get_resource_service('legal_archive')
archive = get_resource_service('archive_publish')
published = get_resource_service('published')
publish_queue = get_resource_service('publish_queue')
self.original_method = LegalArchiveImport.upsert_into_legal_archive
LegalArchiveImport.upsert_into_legal_archive = MagicMock()
for item in self.archive_items:
archive_publish.patch(item['_id'], {'headline': 'publishing', 'abstract': 'publishing'})
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item['_id'])
self.assertIsNone(legal_item, 'Item: {} is not none.'.format(item['_id']))
archive_correct.patch(self.archive_items[1]['_id'], {'headline': 'correcting', 'abstract': 'correcting'})
LegalArchiveImport.upsert_into_legal_archive = self.original_method
self.class_under_test().run(1)
# items are not expired
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item['_id'])
self.assertIsNone(legal_item, 'Item: {} is not none.'.format(item['_id']))
# expire the items
for item in self.archive_items:
original = archive.find_one(req=None, _id=item['_id'])
archive.system_update(item['_id'], {'expiry': utcnow() - timedelta(minutes=30)}, original)
published.update_published_items(item['_id'], 'expiry', utcnow() - timedelta(minutes=30))
# run the command after expiry
self.class_under_test().run(1)
# items are expired
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item['_id'])
self.assertEqual(legal_item['_id'], item['_id'], 'item {} not imported to legal'.format(item['_id']))
# items are moved to legal
for item in self.archive_items:
published_items = list(published.get_other_published_items(item['_id']))
for published_item in published_items:
self.assertEqual(published_item['moved_to_legal'], True)
# items are moved to legal publish queue
for item in self.archive_items:
req = ParsedRequest()
req.where = json.dumps({'item_id': item['_id']})
queue_items = list(publish_queue.get(req=req, lookup=None))
self.assertGreaterEqual(len(queue_items), 1)
for queue_item in queue_items:
self.assertEqual(queue_item['moved_to_legal'], True)
def internal_destination_auto_publish(item, **kwargs):
"""Auto publish the item using internal destination
:param dict item: item to be published
:param kwargs:
:raises StopDuplication: to indicate the superdesk.internal_destination.handle_item_published
to stop duplication as duplication is handle by this method.
"""
if item.get(ITEM_STATE) not in PUBLISH_STATES:
raise InvalidStateTransitionError(message='Internal Destination auto publish macro can '
'only be called after publishing the item.')
operation = item.get(ITEM_OPERATION)
archive_action_service = get_resource_service(publish_services.get(operation))
archive_service = get_resource_service('archive')
extra_fields = [PUBLISH_SCHEDULE, SCHEDULE_SETTINGS]
# if any macro is doing publishing then we need the duplicate item that was published earlier
req = ParsedRequest()
req.where = json.dumps({
'$and': [
{PROCESSED_FROM: item.get(config.ID_FIELD)},
{'task.desk': str(item.get('task').get('desk'))}
]
})
req.max_results = 1
overwrite_item = next((archive_service.get_from_mongo(req=req, lookup=None)), None)
# keep pubslish_schedule and schedule_settings in updates so that state can be set to scheduled
updates = {
PUBLISH_SCHEDULE: item[PUBLISH_SCHEDULE],
SCHEDULE_SETTINGS: item[SCHEDULE_SETTINGS]
}
if item.get(ITEM_STATE) == CONTENT_STATE.PUBLISHED or not overwrite_item:
new_id = archive_service.duplicate_content(item, state='routed', extra_fields=extra_fields)
updates[ITEM_STATE] = item.get(ITEM_STATE)
updates[PROCESSED_FROM] = item[config.ID_FIELD]
get_resource_service('archive_publish').patch(id=new_id, updates=updates)
else:
if overwrite_item:
# get the schema fields
schema_item = apply_schema(deepcopy(item))
keys_to_delete = ['source', 'unique_id', 'unique_name', 'original_id',
'expiry', 'correction_sequence']
# remove the keys
archive_service.remove_after_copy(schema_item, delete_keys=keys_to_delete)
# get the diff
updates.update({key: val for key, val in schema_item.items()
if overwrite_item.get(key) != val and not key.startswith("_")})
archive_action_service.patch(id=overwrite_item[config.ID_FIELD],
updates=updates)
# raise stop duplication on successful completion so that
# internal destination superdesk.internal_destination.handle_item_published
# will not duplicate the item.
raise StopDuplication()
def test_import_into_legal_archive(self):
archive_publish = get_resource_service("archive_publish")
archive_correct = get_resource_service("archive_correct")
legal_archive = get_resource_service("legal_archive")
archive = get_resource_service("archive_publish")
published = get_resource_service("published")
publish_queue = get_resource_service("publish_queue")
self.original_method = LegalArchiveImport.upsert_into_legal_archive
LegalArchiveImport.upsert_into_legal_archive = MagicMock()
for item in self.archive_items:
archive_publish.patch(item["_id"], {"headline": "publishing", "abstract": "publishing"})
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item["_id"])
self.assertIsNone(legal_item, "Item: {} is not none.".format(item["_id"]))
archive_correct.patch(self.archive_items[1]["_id"], {"headline": "correcting", "abstract": "correcting"})
LegalArchiveImport.upsert_into_legal_archive = self.original_method
self.class_under_test().run(1)
# items are not expired
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item["_id"])
self.assertIsNone(legal_item, "Item: {} is not none.".format(item["_id"]))
# expire the items
for item in self.archive_items:
original = archive.find_one(req=None, _id=item["_id"])
archive.system_update(item["_id"], {"expiry": utcnow() - timedelta(minutes=30)}, original)
published.update_published_items(item["_id"], "expiry", utcnow() - timedelta(minutes=30))
# run the command after expiry
self.class_under_test().run(1)
# items are expired
for item in self.archive_items:
legal_item = legal_archive.find_one(req=None, _id=item["_id"])
self.assertEqual(legal_item["_id"], item["_id"], "item {} not imported to legal".format(item["_id"]))
# items are moved to legal
for item in self.archive_items:
published_items = list(published.get_other_published_items(item["_id"]))
for published_item in published_items:
self.assertEqual(published_item["moved_to_legal"], True)
# items are moved to legal publish queue
for item in self.archive_items:
req = ParsedRequest()
req.where = json.dumps({"item_id": item["_id"]})
queue_items = list(publish_queue.get(req=req, lookup=None))
self.assertGreaterEqual(len(queue_items), 1)
for queue_item in queue_items:
self.assertEqual(queue_item["moved_to_legal"], True)
def get(self, req, lookup):
"""Filter out personal activity on personal items if inquired by another user."""
if req is None:
req = ParsedRequest()
user = getattr(g, 'user', None)
if not user:
raise SuperdeskApiError.notFoundError('Can not determine user')
where_cond = {}
if req.where:
if req.where[0] != '{':
req.where = '{' + req.where + '}'
where_cond = json.loads(req.where)
for_user = where_cond.get('user', str(user.get('_id')))
if for_user != str(user.get('_id')):
where_item = {'$and': [{'desk': {'$ne': None}}, {'desk': {'$exists': True}},
{'resource': 'archive'}]}
where_cond['$or'] = [where_item, {'resource': {'$ne': 'archive'}}]
req.where = json.dumps(where_cond)
return self.backend.get(self.datasource, req=req, lookup=lookup)
def get(self, req, lookup):
"""
Overriding because of a different resource URL and user_id is part of the URL
"""
req = ParsedRequest()
req.where = json.dumps({'$or': [lookup, {'is_global': True}]})
items = list(super().get(req, lookup=None))
for item in items:
item['filter'] = json.loads(item.get('filter'))
return ListCursor(items)
def get(self, req, lookup):
"""
Overriding because of a different resource URL and user_id is part of the URL
"""
req = ParsedRequest()
req.where = json.dumps({'$or': [lookup, {'is_global': True}]})
items = list(super().get(req, lookup=None))
for item in items:
item['filter'] = decode_filter(item.get('filter'))
return ListCursor(items)
def find(self, endpoint_name, where, max_results=0):
"""Find items for given endpoint using mongo query in python dict object.
It handles request creation here so no need to do this in service.
:param string endpoint_name
:param dict where
:param int max_results
"""
req = ParsedRequest()
req.where = MongoJSONEncoder().encode(where)
req.max_results = max_results
return self.get_from_mongo(endpoint_name, req, None)
def find(self, endpoint_name, where, max_results=0):
"""Find items for given endpoint using mongo query in python dict object.
It handles request creation here so no need to do this in service.
:param string endpoint_name
:param dict where
:param int max_results
"""
req = ParsedRequest()
req.where = MongoJSONEncoder().encode(where)
req.max_results = max_results
return self.get_from_mongo(endpoint_name, req, None)
def get_expired_items(self,
expiry_datetime=None,
expiry_days=None,
max_results=None,
include_children=True):
"""Get the expired items.
Returns a generator for the list of expired items, sorting by `_id` and returning `max_results` per iteration.
:param datetime expiry_datetime: Expiry date/time used to retrieve the list of items, defaults to `utcnow()`
:param int expiry_days: Number of days content expires, defaults to `CONTENT_API_EXPIRY_DAYS`
:param int max_results: Maximum results to retrieve per iteration, defaults to `MAX_EXPIRY_QUERY_LIMIT`
:param boolean include_children: Include only root item if False, otherwise include the entire item chain
:return list: expired content_api items
"""
if expiry_datetime is None:
expiry_datetime = utcnow()
if expiry_days is None:
expiry_days = app.settings["CONTENT_API_EXPIRY_DAYS"]
if max_results is None:
max_results = app.settings["MAX_EXPIRY_QUERY_LIMIT"]
last_id = None
expire_at = date_to_str(expiry_datetime - timedelta(days=expiry_days))
while True:
query = {"$and": [{"_updated": {"$lte": expire_at}}]}
if last_id is not None:
query["$and"].append({"_id": {"$gt": last_id}})
if not include_children:
query["$and"].append({"ancestors": {"$exists": False}})
req = ParsedRequest()
req.sort = "_id"
req.where = json.dumps(query)
req.max_results = max_results
items = list(self.get_from_mongo(req=req, lookup=None))
if not items:
break
last_id = items[-1]["_id"]
yield items
def get_expired_items(self, expiry_datetime, invalid_only=False):
"""Get the expired items.
Where content state is not scheduled and the item matches given parameters
:param datetime expiry_datetime: expiry datetime
:param bool invalid_only: True only invalid items
:return pymongo.cursor: expired non published items.
"""
unique_id = 0
while True:
req = ParsedRequest()
req.sort = 'unique_id'
query = {
'$and': [{
'expiry': {
'$lte': date_to_str(expiry_datetime)
}
}, {
'$or': [{
'task.desk': {
'$ne': None
}
}, {
ITEM_STATE: CONTENT_STATE.SPIKED,
'task.desk': None
}]
}]
}
query['$and'].append({'unique_id': {'$gt': unique_id}})
if invalid_only:
query['$and'].append({'expiry_status': 'invalid'})
else:
query['$and'].append({'expiry_status': {'$ne': 'invalid'}})
req.where = json.dumps(query)
req.max_results = config.MAX_EXPIRY_QUERY_LIMIT
items = list(self.get_from_mongo(req=req, lookup=None))
if not len(items):
break
unique_id = items[-1]['unique_id']
yield items
def get_expired_items(self, expiry_datetime=None, expiry_days=None, max_results=None, include_children=True):
"""Get the expired items.
Returns a generator for the list of expired items, sorting by `_id` and returning `max_results` per iteration.
:param datetime expiry_datetime: Expiry date/time used to retrieve the list of items, defaults to `utcnow()`
:param int expiry_days: Number of days content expires, defaults to `CONTENT_API_EXPIRY_DAYS`
:param int max_results: Maximum results to retrieve per iteration, defaults to `MAX_EXPIRY_QUERY_LIMIT`
:param boolean include_children: Include only root item if False, otherwise include the entire item chain
:return list: expired content_api items
"""
if expiry_datetime is None:
expiry_datetime = utcnow()
if expiry_days is None:
expiry_days = app.settings['CONTENT_API_EXPIRY_DAYS']
if max_results is None:
max_results = app.settings['MAX_EXPIRY_QUERY_LIMIT']
last_id = None
expire_at = date_to_str(expiry_datetime - timedelta(days=expiry_days))
while True:
query = {'$and': [{'_updated': {'$lte': expire_at}}]}
if last_id is not None:
query['$and'].append({'_id': {'$gt': last_id}})
if not include_children:
query['$and'].append({'ancestors': {'$exists': False}})
req = ParsedRequest()
req.sort = '_id'
req.where = json.dumps(query)
req.max_results = max_results
items = list(self.get_from_mongo(req=req, lookup=None))
if not items:
break
last_id = items[-1]['_id']
yield items
def hook_on_updated(resource, updates, original):
""" called after to run on update operation """
config = get_config(resource)
# couple of situations where we have to do nothing because
# `PATH_BUILD_FROM` field has not changed
if config['PATH_BUILD_FROM'] not in updates:
return
if updates[config['PATH_BUILD_FROM']] ==\
original[config['PATH_BUILD_FROM']]:
return
# get all children that belong to the path of the document
path = original[config['PATH']] +\
original[config['PATH_BUILD_FROM']] +\
config['SEPARATOR']
r = ParsedRequest()
r.where = dumps({config['PATH']: {'$regex': "^{}".format(path)}})
r.max_resuls = 0
children = current_app.data.find(resource, r, None)
# Then update each child, where the old path has to be updated
# with the right one.
path_build_from = updates[config['PATH_BUILD_FROM']]
level_path_parent = len(original[config['PATH']].split(".")) - 1\
if original[config['PATH']] else 0
for child in children:
# build the new path using the old path as a first part
# and the last part, only replacement of the field value changed
old_path = child[config['PATH']].split(config['SEPARATOR']) or []
new_path = config['SEPARATOR'].join(
old_path[:level_path_parent] +
[path_build_from] +
old_path[level_path_parent+1:])
# finally udpate the document
current_app.data.update(
resource,
child[current_app.config['ID_FIELD']],
{config['PATH']: new_path},
child
)
def get_desk():
"""
Search for a desk on the system with the name "Copytakers"
:return:
"""
logger.info('Fetching the ObjectID for the desk "Copytakers".')
query = {'name': 'Copytakers'}
req = ParsedRequest()
req.where = json.dumps(query)
desk_service = get_resource_service('desks')
desk_item = list(desk_service.get_from_mongo(req=req, lookup=None))
if not desk_item:
raise ('Failed to find the a desk called "Copytakers".')
desk_id = desk_item[0]['_id']
logger.info('ObjectID for the desk Copytakers is {}.'.format(desk_id))
return desk_item[0]
def _get_future_events(self, event):
"""Utility method to get future events for the supplied event"""
query = {
'$and': [
# All the events created from the same recurring rules
{'recurrence_id': event['recurrence_id']},
# Only future events
{'dates.start': {'$gt': date_to_str(event['dates']['start'])}},
# Except the provided event
{'_id': {'$ne': event[config.ID_FIELD]}}
]
}
req = ParsedRequest()
req.sort = '[("dates.start", 1)]'
req.where = json.dumps(query)
return self.get_from_mongo(req, {})
def _get_desk_id(self):
"""Returns the ObjectID of the desk
:return str: The ObjectID for the desk provided, or None if the desk was not found
"""
logger.info('Fetching the ObjectID for the desk {}.'.format(self.default_desk))
query = {'name': self.default_desk}
req = ParsedRequest()
req.where = json.dumps(query)
desk_service = get_resource_service('desks')
desk_item = list(desk_service.get_from_mongo(req=req, lookup=None))
if not desk_item:
logger.error('Failed to find the ObjectID for the provided desk {}'.format(self.default_desk))
return None
desk_id = desk_item[0]['_id']
logger.info('ObjectID for the desk {} is {}.'.format(self.default_desk, desk_id))
return desk_id
def get_expired_items(self, expiry_datetime, invalid_only=False):
"""Get the expired items.
Where content state is not scheduled and the item matches given parameters
:param datetime expiry_datetime: expiry datetime
:param bool invalid_only: True only invalid items
:return pymongo.cursor: expired non published items.
"""
unique_id = 0
while True:
req = ParsedRequest()
req.sort = 'unique_id'
query = {
'$and': [
{'expiry': {'$lte': date_to_str(expiry_datetime)}},
{'$or': [
{'task.desk': {'$ne': None}},
{ITEM_STATE: CONTENT_STATE.SPIKED, 'task.desk': None}
]}
]
}
query['$and'].append({'unique_id': {'$gt': unique_id}})
if invalid_only:
query['$and'].append({'expiry_status': 'invalid'})
else:
query['$and'].append({'expiry_status': {'$ne': 'invalid'}})
req.where = json.dumps(query)
req.max_results = config.MAX_EXPIRY_QUERY_LIMIT
items = list(self.get_from_mongo(req=req, lookup=None))
if not len(items):
break
unique_id = items[-1]['unique_id']
yield items
def get_series(self, query, sort, max_results):
page = 1
while True:
# Get the results from mongo
req = ParsedRequest()
req.sort = sort
req.where = json.dumps(query)
req.max_results = max_results
req.page = page
results = self.get_from_mongo(req=req, lookup=None)
docs = list(results)
if not docs:
break
page += 1
# Yield the results for iteration by the callee
for doc in docs:
yield doc
def test_search_capi(self):
subscriber = {'_id': 'sub1'}
self.content_api.publish({'_id': 'foo', 'guid': 'foo', 'type': 'text',
'anpa_category': [{'qcode': 'i', 'name': 'International News'}],
'headline': 'Man bites dog'}, [subscriber])
self.content_api.publish({'_id': 'bar', 'guid': 'bar', 'type': 'text'}, [{'_id': 'sub2'}])
test = superdesk.get_resource_service('search_capi')
req = ParsedRequest()
req.args = MultiDict([('subscribers', 'sub1')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
resp = test.get(req=None, lookup=None)
self.assertEqual(resp.count(), 2)
req = ParsedRequest()
req.args = MultiDict()
req.where = '{"headline":"dog"}'
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
def get(self, req, lookup):
"""Retrieve a list of items that match the filter criteria (if any)
pssed along the HTTP request.
:param req: object representing the HTTP request
:type req: `eve.utils.ParsedRequest`
:param dict lookup: sub-resource lookup from the endpoint URL
:return: database results cursor object
:rtype: `pymongo.cursor.Cursor`
"""
if req is None:
req = ParsedRequest()
allowed_params = (
'q', 'start_date', 'end_date',
'include_fields', 'exclude_fields'
)
self._check_for_unknown_params(req, whitelist=allowed_params)
# set the "q" filter
request_params = req.args or {}
query_filter = {}
if 'q' in request_params:
# TODO: add validation for the "q" parameter when we define its
# format and implement the corresponding actions
query_filter = json.loads(request_params['q'])
# set the date range filter
start_date, end_date = self._get_date_range(request_params)
date_filter = self._create_date_range_filter(start_date, end_date)
query_filter.update(date_filter)
req.where = json.dumps(query_filter)
self._set_fields_filter(req) # Eve's "projection"
return super().get(req, lookup)
def test_search_capi(self):
subscriber = {'_id': 'sub1'}
self.content_api.publish({'_id': 'foo', 'guid': 'foo', 'type': 'text',
'anpa_category': [{'qcode': 'i', 'name': 'International News'}],
'headline': 'Man bites dog'}, [subscriber])
self.content_api.publish({'_id': 'bar', 'guid': 'bar', 'type': 'text'}, [{'_id': 'sub2'}])
test = superdesk.get_resource_service('search_capi')
req = ParsedRequest()
req.args = MultiDict([('subscribers', 'sub1')])
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
resp = test.get(req=None, lookup=None)
self.assertEqual(resp.count(), 2)
req = ParsedRequest()
req.args = MultiDict()
req.where = '{"headline":"dog"}'
resp = test.get(req=req, lookup=None)
self.assertEqual(resp.count(), 1)
def _get_field_values(self):
values = {}
vocabularies_resource = get_resource_service("vocabularies")
values["anpa_category"] = vocabularies_resource.find_one(req=None, _id="categories")["items"]
req = ParsedRequest()
req.where = json.dumps({"$or": [{"schema_field": "genre"}, {"_id": "genre"}]})
genre = vocabularies_resource.get(req=req, lookup=None)
if genre.count():
values["genre"] = genre[0]["items"]
values["urgency"] = vocabularies_resource.find_one(req=None, _id="urgency")["items"]
values["priority"] = vocabularies_resource.find_one(req=None, _id="priority")["items"]
values["type"] = vocabularies_resource.find_one(req=None, _id="type")["items"]
subject = vocabularies_resource.find_one(req=None, schema_field="subject")
if subject:
values["subject"] = subject["items"]
else:
values["subject"] = get_subjectcodeitems()
values["desk"] = list(get_resource_service("desks").get(None, {}))
values["stage"] = self._get_stage_field_values(values["desk"])
values["sms"] = [{"qcode": 0, "name": "False"}, {"qcode": 1, "name": "True"}]
values["place"] = vocabularies_resource.find_one(req=None, _id="locators")["items"]
return values
def _get_desk_id(self):
"""Returns the ObjectID of the desk
:return str: The ObjectID for the desk provided, or None if the desk was not found
"""
logger.info('Fetching the ObjectID for the desk {}.'.format(
self.default_desk))
query = {'name': self.default_desk}
req = ParsedRequest()
req.where = json.dumps(query)
desk_service = get_resource_service('desks')
desk_item = list(desk_service.get_from_mongo(req=req, lookup=None))
if not desk_item:
logger.error(
'Failed to find the ObjectID for the provided desk {}'.format(
self.default_desk))
return None
desk_id = desk_item[0]['_id']
logger.info('ObjectID for the desk {} is {}.'.format(
self.default_desk, desk_id))
return desk_id
def get(self, req, lookup):
"""
Overriding to pass user as search parameter
"""
session_user = str(get_user_id(required=True))
if not req:
req = ParsedRequest()
where = json.loads(req.where) if req.where else {}
if lookup:
where.update(lookup)
if '$or' not in where:
where['$or'] = []
# Restrict the saved reports to either global or owned by current user
where['$or'].extend([{'is_global': True}, {'user': session_user}])
req.where = json.dumps(where)
return super().get(req, lookup=None)
def get_recurring_timeline(self, selected):
"""Utility method to get all events in the series
This splits up the series of events into 3 separate arrays.
Historic: event.dates.start < utcnow()
Past: utcnow() < event.dates.start < selected.dates.start
Future: event.dates.start > selected.dates.start
"""
historic = []
past = []
future = []
selected_start = selected.get('dates', {}).get('start', utcnow())
req = ParsedRequest()
req.sort = '[("dates.start", 1)]'
req.where = json.dumps({
'$and': [{
'recurrence_id': selected['recurrence_id']
}, {
'_id': {
'$ne': selected[config.ID_FIELD]
}
}]
})
for event in list(self.get_from_mongo(req, {})):
end = event['dates']['end']
start = event['dates']['start']
if end < utcnow():
historic.append(event)
elif start < selected_start:
past.append(event)
elif start > selected_start:
future.append(event)
return historic, past, future
