def update(self, id, updates, original):
original_state = original[ITEM_STATE]
if not is_workflow_state_transition_valid('spike', original_state):
raise InvalidStateTransitionError()
package_service = PackageService()
user = get_user(required=True)
item = get_resource_service(ARCHIVE).find_one(req=None, _id=id)
expiry_minutes = app.settings['SPIKE_EXPIRY_MINUTES']
# check if item is in a desk. If it's then use the desks spike_expiry
if is_assigned_to_a_desk(item):
desk = get_resource_service('desks').find_one(
_id=item['task']['desk'], req=None)
expiry_minutes = desk.get('spike_expiry', expiry_minutes)
updates[EXPIRY] = get_expiry_date(expiry_minutes)
updates[REVERT_STATE] = item.get(ITEM_STATE, None)
if original.get('rewrite_of'):
updates['rewrite_of'] = None
item = self.backend.update(self.datasource, id, updates, original)
push_notification('item:spike',
item=str(item.get('_id')),
user=str(user))
package_service.remove_spiked_refs_from_package(id)
return item
def update(self, id, updates, original):
original_state = original[ITEM_STATE]
if not is_workflow_state_transition_valid('spike', original_state):
raise InvalidStateTransitionError()
package_service = PackageService()
user = get_user(required=True)
item = get_resource_service(ARCHIVE).find_one(req=None, _id=id)
expiry_minutes = app.settings['SPIKE_EXPIRY_MINUTES']
# check if item is in a desk. If it's then use the desks spike_expiry
if is_assigned_to_a_desk(item):
desk = get_resource_service('desks').find_one(_id=item['task']['desk'], req=None)
expiry_minutes = desk.get('spike_expiry', expiry_minutes)
updates[EXPIRY] = get_expiry_date(expiry_minutes)
updates[REVERT_STATE] = item.get(ITEM_STATE, None)
if original.get('rewrite_of'):
updates['rewrite_of'] = None
item = self.backend.update(self.datasource, id, updates, original)
push_notification('item:spike', item=str(item.get('_id')), user=str(user))
package_service.remove_spiked_refs_from_package(id)
return item
def update(self, id, updates, original):
original_state = original[config.CONTENT_STATE]
if not is_workflow_state_transition_valid("spike", original_state):
raise InvalidStateTransitionError()
package_service = PackageService()
user = get_user(required=True)
item = get_resource_service(ARCHIVE).find_one(req=None, _id=id)
expiry_minutes = app.settings["SPIKE_EXPIRY_MINUTES"]
# check if item is in a desk. If it's then use the desks spike_expiry
if is_assigned_to_a_desk(item):
desk = get_resource_service("desks").find_one(_id=item["task"]["desk"], req=None)
expiry_minutes = desk.get("spike_expiry", expiry_minutes)
updates[EXPIRY] = get_expiry_date(expiry_minutes)
updates[REVERT_STATE] = item.get(app.config["CONTENT_STATE"], None)
if original.get("rewrite_of"):
updates["rewrite_of"] = None
item = self.backend.update(self.datasource, id, updates, original)
push_notification("item:spike", item=str(item.get("_id")), user=str(user))
package_service.remove_spiked_refs_from_package(id)
return item
def update(self, id, updates, original):
original_state = original[ITEM_STATE]
if not is_workflow_state_transition_valid(ITEM_SPIKE, original_state):
raise InvalidStateTransitionError()
user = get_user(required=True)
item = get_resource_service(ARCHIVE).find_one(req=None, _id=id)
task = item.get('task', {})
updates[EXPIRY] = self._get_spike_expiry(desk_id=task.get('desk'), stage_id=task.get('stage'))
updates[REVERT_STATE] = item.get(ITEM_STATE, None)
if original.get('rewrite_of'):
updates['rewrite_of'] = None
if original.get('rewritten_by'):
updates['rewritten_by'] = None
if original.get('broadcast'):
updates['broadcast'] = None
if original.get('rewrite_sequence'):
updates['rewrite_sequence'] = None
# remove any relation with linked items
updates[ITEM_EVENT_ID] = generate_guid(type=GUID_TAG)
# remove lock
updates.update({
'lock_user': None,
'lock_session': None,
})
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
# remove links from items in the package
package_service = PackageService()
items = package_service.get_item_refs(original)
for item in items:
package_item = get_resource_service(ARCHIVE).find_one(req=None, _id=item[GUID_FIELD])
if package_item:
linked_in_packages = [linked for linked in package_item.get(LINKED_IN_PACKAGES, [])
if linked.get(PACKAGE) != original.get(config.ID_FIELD)]
super().system_update(package_item[config.ID_FIELD],
{LINKED_IN_PACKAGES: linked_in_packages},
package_item)
# keep the structure of old group in order to be able to unspike the package
updates[DELETED_GROUPS] = original[GROUPS]
# and remove all the items from the package
updates['groups'] = []
item = self.backend.update(self.datasource, id, updates, original)
push_notification('item:spike', item=str(id), user=str(user.get(config.ID_FIELD)))
history_updates = dict(updates)
if original.get('task'):
history_updates['task'] = original.get('task')
app.on_archive_item_updated(history_updates, original, ITEM_SPIKE)
self._removed_refs_from_package(id)
return item
def update(self, id, updates, original):
original_state = original[ITEM_STATE]
if not is_workflow_state_transition_valid(ITEM_SPIKE, original_state):
raise InvalidStateTransitionError()
user = get_user(required=True)
item = get_resource_service(ARCHIVE).find_one(req=None, _id=id)
task = item.get('task', {})
updates[EXPIRY] = self._get_spike_expiry(desk_id=task.get('desk'),
stage_id=task.get('stage'))
updates[REVERT_STATE] = item.get(ITEM_STATE, None)
if original.get('rewrite_of'):
updates['rewrite_of'] = None
if original.get('rewritten_by'):
updates['rewritten_by'] = None
if original.get('broadcast'):
updates['broadcast'] = None
if original.get('rewrite_sequence'):
updates['rewrite_sequence'] = None
# remove any relation with linked items
updates[ITEM_EVENT_ID] = generate_guid(type=GUID_TAG)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
# remove links from items in the package
package_service = PackageService()
items = package_service.get_item_refs(original)
for item in items:
package_item = get_resource_service(ARCHIVE).find_one(
req=None, _id=item[GUID_FIELD])
if package_item:
linked_in_packages = [
linked
for linked in package_item.get(LINKED_IN_PACKAGES, [])
if linked.get(PACKAGE) != original.get(config.ID_FIELD)
]
super().system_update(
package_item[config.ID_FIELD],
{LINKED_IN_PACKAGES: linked_in_packages}, package_item)
# and remove all the items from the package
updates['groups'] = []
item = self.backend.update(self.datasource, id, updates, original)
push_notification('item:spike',
item=str(id),
user=str(user.get(config.ID_FIELD)))
history_updates = dict(updates)
if original.get('task'):
history_updates['task'] = original.get('task')
app.on_archive_item_updated(history_updates, original, ITEM_SPIKE)
self._removed_refs_from_package(id)
return item
def _can_remove_item(self, item, processed_item=None):
"""Recursively checks if the item can be removed.
:param dict item: item to be remove
:param set processed_item: processed items
:return: True if item can be removed, False otherwise.
"""
if processed_item is None:
processed_item = dict()
item_refs = []
package_service = PackageService()
archive_service = get_resource_service(ARCHIVE)
if item.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
# Get the item references for is package
item_refs = package_service.get_residrefs(item)
if item.get(ITEM_TYPE) in [CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED]:
broadcast_items = get_resource_service('archive_broadcast').get_broadcast_items_from_master_story(item)
# If master story expires then check if broadcast item is included in a package.
# If included in a package then check the package expiry.
item_refs.extend([broadcast_item.get(config.ID_FIELD) for broadcast_item in broadcast_items])
if item.get('rewrite_of'):
item_refs.append(item.get('rewrite_of'))
if item.get('rewritten_by'):
item_refs.append(item.get('rewritten_by'))
# get the list of associated item ids
if item.get(ITEM_TYPE) in MEDIA_TYPES:
item_refs.extend(self._get_associated_items(item))
else:
item_refs.extend(self._get_associated_media_id(item))
# get item reference where this referred
item_refs.extend(package_service.get_linked_in_package_ids(item))
# check item refs in the ids to remove set
is_expired = item.get('expiry') and item.get('expiry') < utcnow()
if is_expired:
# now check recursively for all references
if item.get(config.ID_FIELD) in processed_item:
return is_expired
processed_item[item.get(config.ID_FIELD)] = item
if item_refs:
archive_items = archive_service.get_from_mongo(req=None, lookup={'_id': {'$in': item_refs}})
for archive_item in archive_items:
is_expired = self._can_remove_item(archive_item, processed_item)
if not is_expired:
break
return is_expired
def _can_remove_item(self, item, processed_item=None):
"""
Recursively checks if the item can be removed.
:param dict item: item to be remove
:param set processed_item: processed items
:return: True if item can be removed, False otherwise.
"""
if processed_item is None:
processed_item = set()
item_refs = []
package_service = PackageService()
archive_service = get_resource_service(ARCHIVE)
if item.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
# Get the item references for is package
item_refs = package_service.get_residrefs(item)
if item.get(PACKAGE_TYPE) == TAKES_PACKAGE or \
item.get(ITEM_TYPE) in [CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED]:
broadcast_items = get_resource_service(
'archive_broadcast').get_broadcast_items_from_master_story(
item)
# If master story expires then check if broadcast item is included in a package.
# If included in a package then check the package expiry.
item_refs.extend([
broadcast_item.get(config.ID_FIELD)
for broadcast_item in broadcast_items
])
# get item reference where this referred
item_refs.extend(package_service.get_linked_in_package_ids(item))
# check item refs in the ids to remove set
is_expired = item.get('expiry') < utcnow()
if is_expired:
# now check recursively for all references
if item.get(config.ID_FIELD) in processed_item:
return is_expired
processed_item.add(item.get(config.ID_FIELD))
if item_refs:
archive_items = archive_service.get_from_mongo(
req=None, lookup={'_id': {
'$in': item_refs
}})
for archive_item in archive_items:
is_expired = self._can_remove_item(archive_item,
processed_item)
if not is_expired:
break
return is_expired
def on_updated(self, updates, original):
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE and updates.get(GROUPS, None):
# restore the deleted items from package
package_service = PackageService()
items = package_service.get_item_refs(updates)
for item in items:
package_item = get_resource_service(ARCHIVE).find_one(req=None, _id=item[GUID_FIELD])
if package_item:
linked_in_packages = [linked for linked in package_item.get(LINKED_IN_PACKAGES, [])
if linked.get(PACKAGE) != original.get(config.ID_FIELD)]
linked_in_packages.append({PACKAGE: original.get(config.ID_FIELD)})
super().system_update(package_item[config.ID_FIELD],
{LINKED_IN_PACKAGES: linked_in_packages},
package_item)
def on_create(self, docs):
package_service = PackageService()
for doc in docs:
doc.pop('lock_user', None)
doc.pop('lock_time', None)
doc.pop('lock_action', None)
doc.pop('lock_session', None)
doc.pop('highlights', None)
doc.pop('marked_desks', None)
doc['archived_id'] = self._get_archived_id(doc.get('item_id'), doc.get(config.VERSION))
if doc.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
for ref in package_service.get_item_refs(doc):
ref['location'] = 'archived'
def on_create(self, docs):
package_service = PackageService()
for doc in docs:
doc.pop("lock_user", None)
doc.pop("lock_time", None)
doc.pop("lock_action", None)
doc.pop("lock_session", None)
doc.pop("highlights", None)
doc.pop("marked_desks", None)
doc["archived_id"] = self._get_archived_id(doc.get("item_id"),
doc.get(config.VERSION))
if doc.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
for ref in package_service.get_item_refs(doc):
ref["location"] = "archived"
def on_updated(self, updates, original):
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE and updates.get(GROUPS, None):
# restore the deleted items from package
package_service = PackageService()
items = package_service.get_item_refs(updates)
for item in items:
package_item = get_resource_service(ARCHIVE).find_one(req=None, _id=item[GUID_FIELD])
if package_item:
linked_in_packages = [
linked
for linked in package_item.get(LINKED_IN_PACKAGES, [])
if linked.get(PACKAGE) != original.get(config.ID_FIELD)
]
linked_in_packages.append({PACKAGE: original.get(config.ID_FIELD)})
super().system_update(
package_item[config.ID_FIELD], {LINKED_IN_PACKAGES: linked_in_packages}, package_item
)
def update(self, id, updates, original):
original_state = original[ITEM_STATE]
if not is_workflow_state_transition_valid('spike', original_state):
raise InvalidStateTransitionError()
user = get_user(required=True)
item = get_resource_service(ARCHIVE).find_one(req=None, _id=id)
task = item.get('task', {})
updates[EXPIRY] = self._get_spike_expiry(desk_id=task.get('desk'), stage_id=task.get('stage'))
updates[REVERT_STATE] = item.get(ITEM_STATE, None)
if original.get('rewrite_of'):
updates['rewrite_of'] = None
if original.get('rewritten_by'):
updates['rewritten_by'] = None
if original.get('broadcast'):
updates['broadcast'] = None
if original.get('rewrite_sequence'):
updates['rewrite_sequence'] = None
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
# remove links from items in the package
package_service = PackageService()
items = package_service.get_item_refs(original)
for item in items:
package_item = get_resource_service(ARCHIVE).find_one(req=None, _id=item['guid'])
if package_item:
linked_in_packages = [linked for linked in package_item.get(LINKED_IN_PACKAGES, [])
if linked.get(PACKAGE) != original.get(config.ID_FIELD)]
super().system_update(package_item[config.ID_FIELD],
{LINKED_IN_PACKAGES: linked_in_packages},
package_item)
# and remove all the items from the package
updates['groups'] = []
item = self.backend.update(self.datasource, id, updates, original)
push_notification('item:spike', item=str(id), user=str(user.get(config.ID_FIELD)))
self._removed_refs_from_package(id)
return item
def _remove_and_set_kill_properties(self, article, articles_to_kill,
updates):
"""Removes the irrelevant properties from the given article and sets the properties for kill operation.
:param article: article from the archived repo
:type article: dict
:param articles_to_kill: list of articles which were about to kill from dusty archive
:type articles_to_kill: list
:param updates: updates to be applied on the article before saving
:type updates: dict
"""
article.pop("archived_id", None)
article.pop("_type", None)
article.pop("_links", None)
article.pop("queue_state", None)
article.pop(config.ETAG, None)
for field in ["headline", "abstract", "body_html"]:
article[field] = updates.get(field, article.get(field, ""))
article[ITEM_STATE] = CONTENT_STATE.KILLED if updates[
ITEM_OPERATION] == ITEM_KILL else CONTENT_STATE.RECALLED
article[ITEM_OPERATION] = updates[ITEM_OPERATION]
article["pubstatus"] = PUB_STATUS.CANCELED
article[config.LAST_UPDATED] = utcnow()
user = get_user()
article["version_creator"] = str(user[config.ID_FIELD])
resolve_document_version(article, ARCHIVE, "PATCH", article)
if article[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
package_service = PackageService()
item_refs = package_service.get_item_refs(article)
for ref in item_refs:
item_in_package = [
item for item in articles_to_kill if item.get(
"item_id", item.get(config.ID_FIELD)) == ref[RESIDREF]
]
ref["location"] = ARCHIVE
ref[config.VERSION] = item_in_package[0][config.VERSION]
def on_create(self, docs):
package_service = PackageService()
for doc in docs:
doc.pop('lock_user', None)
doc.pop('lock_time', None)
doc.pop('lock_session', None)
doc['archived_id'] = self._get_archived_id(doc.get('item_id'), doc.get(config.VERSION))
if doc.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
is_takes_package = doc.get(PACKAGE_TYPE) == TAKES_PACKAGE
for ref in package_service.get_item_refs(doc):
ref['location'] = 'archived'
if is_takes_package and not ref.get('is_published'):
# if take is not published
package_service.remove_ref_from_inmem_package(doc, ref.get(RESIDREF))
if is_takes_package:
doc[SEQUENCE] = len(package_service.get_item_refs(doc))
def _remove_and_set_kill_properties(self, article, articles_to_kill, updates):
"""
Removes the irrelevant properties from the given article and sets the properties for kill operation.
:param article: article from the archived repo
:type article: dict
:param articles_to_kill: list of articles which were about to kill from dusty archive
:type articles_to_kill: list
:param updates: updates to be applied on the article before saving
:type updates: dict
"""
article.pop('archived_id', None)
article.pop('_type', None)
article.pop('_links', None)
article.pop('queue_state', None)
article.pop(config.ETAG, None)
for field in ['headline', 'abstract', 'body_html']:
article[field] = updates.get(field, article.get(field, ''))
article[ITEM_STATE] = CONTENT_STATE.KILLED
article[ITEM_OPERATION] = ITEM_KILL
article['pubstatus'] = PUB_STATUS.CANCELED
article[config.LAST_UPDATED] = utcnow()
user = get_user()
article['version_creator'] = str(user[config.ID_FIELD])
resolve_document_version(article, ARCHIVE, 'PATCH', article)
if article[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
package_service = PackageService()
item_refs = package_service.get_item_refs(article)
for ref in item_refs:
item_in_package = [item for item in articles_to_kill
if item.get('item_id', item.get(config.ID_FIELD)) == ref[RESIDREF]]
ref['location'] = ARCHIVE
ref[config.VERSION] = item_in_package[0][config.VERSION]
def delete(self, lookup):
target_id = request.view_args['target_id']
archive_service = get_resource_service(ARCHIVE)
target = archive_service.find_one(req=None, _id=target_id)
self._validate_unlink(target)
updates = {}
takes_package = TakesPackageService().get_take_package(target)
if takes_package and TakesPackageService().is_last_takes_package_item(target):
# remove the take link
PackageService().remove_refs_in_package(takes_package, target_id)
if target.get('rewrite_of'):
# remove the rewrite info
ArchiveSpikeService().update_rewrite(target)
if not takes_package and not target.get('rewrite_of'):
# there is nothing to do
raise SuperdeskApiError.badRequestError("Only takes and updates can be unlinked!")
if target.get('rewrite_of'):
updates['rewrite_of'] = None
if target.get('anpa_take_key'):
updates['anpa_take_key'] = None
if target.get('rewrite_sequence'):
updates['rewrite_sequence'] = None
if target.get('sequence'):
updates['sequence'] = None
updates['event_id'] = generate_guid(type=GUID_TAG)
archive_service.system_update(target_id, updates, target)
user = get_user(required=True)
push_notification('item:unlink', item=target_id, user=str(user.get(config.ID_FIELD)))
app.on_archive_item_updated(updates, target, ITEM_UNLINK)
def on_create(self, docs):
package_service = PackageService()
for doc in docs:
doc.pop('lock_user', None)
doc.pop('lock_time', None)
doc.pop('lock_session', None)
doc['archived_id'] = self._get_archived_id(doc.get('item_id'),
doc.get(config.VERSION))
if doc.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
is_takes_package = doc.get(PACKAGE_TYPE) == TAKES_PACKAGE
for ref in package_service.get_item_refs(doc):
ref['location'] = 'archived'
if is_takes_package and not ref.get('is_published'):
# if take is not published
package_service.remove_ref_from_inmem_package(
doc, ref.get(RESIDREF))
if is_takes_package:
doc[SEQUENCE] = len(package_service.get_item_refs(doc))
def update(self, id, updates, original):
original_state = original[ITEM_STATE]
if not is_workflow_state_transition_valid(ITEM_SPIKE, original_state):
raise InvalidStateTransitionError()
archive_service = get_resource_service(ARCHIVE)
published_service = get_resource_service("published")
user = get_user(required=True)
item = archive_service.find_one(req=None, _id=id)
task = item.get("task", {})
updates[EXPIRY] = self._get_spike_expiry(desk_id=task.get("desk"),
stage_id=task.get("stage"))
updates[REVERT_STATE] = item.get(ITEM_STATE, None)
if original.get("rewrite_of"):
updates["rewrite_of"] = None
if original.get("rewritten_by"):
updates["rewritten_by"] = None
if original.get("broadcast"):
updates["broadcast"] = None
if original.get("rewrite_sequence"):
updates["rewrite_sequence"] = None
if original.get("marked_for_user"):
# remove marked_for_user on spike and keep it as previous_marked_user for history
updates["previous_marked_user"] = original["marked_for_user"]
updates["marked_for_user"] = None
updates["marked_for_sign_off"] = None
if original.get("translation_id") and original.get("translated_from"):
# remove translations info from the translated item on spike
updates["translated_from"] = None
updates["translation_id"] = None
id_to_remove = original.get(config.ID_FIELD)
# Remove the translated item from the list of translations in the original item
# where orignal item can be in archive or in both archive and published resource as well
translated_from = archive_service.find_one(
req=None, _id=original.get("translated_from"))
translated_from_id = translated_from.get(config.ID_FIELD)
self._remove_translations(archive_service, translated_from,
id_to_remove)
if translated_from.get("state") in PUBLISH_STATES:
published_items = list(
published_service.get_from_mongo(
req=None, lookup={"item_id": translated_from_id}))
if published_items:
for item in published_items:
self._remove_translations(published_service, item,
id_to_remove)
# remove any relation with linked items
updates[ITEM_EVENT_ID] = generate_guid(type=GUID_TAG)
# remove lock
updates.update({
"lock_user": None,
"lock_session": None,
})
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
# remove links from items in the package
package_service = PackageService()
items = package_service.get_item_refs(original)
for item in items:
package_item = archive_service.find_one(req=None,
_id=item[GUID_FIELD])
if package_item:
linked_in_packages = [
linked
for linked in package_item.get(LINKED_IN_PACKAGES, [])
if linked.get(PACKAGE) != original.get(config.ID_FIELD)
]
super().system_update(
package_item[config.ID_FIELD],
{LINKED_IN_PACKAGES: linked_in_packages}, package_item)
# keep the structure of old group in order to be able to unspike the package
updates[DELETED_GROUPS] = original[GROUPS]
# and remove all the items from the package
updates["groups"] = []
item = self.backend.update(self.datasource, id, updates, original)
push_notification("item:spike",
item=str(id),
user=str(user.get(config.ID_FIELD)))
history_updates = dict(updates)
if original.get("task"):
history_updates["task"] = original.get("task")
app.on_archive_item_updated(history_updates, original, ITEM_SPIKE)
self._removed_refs_from_package(id)
return item
class ArchiveService(BaseService):
packageService = PackageService()
takesService = TakesPackageService()
mediaService = ArchiveMediaService()
def on_fetched(self, docs):
"""
Overriding this to handle existing data in Mongo & Elastic
"""
self.__enhance_items(docs[config.ITEMS])
def on_fetched_item(self, doc):
self.__enhance_items([doc])
def __enhance_items(self, items):
for item in items:
handle_existing_data(item)
self.takesService.enhance_with_package_info(item)
def on_create(self, docs):
on_create_item(docs)
for doc in docs:
doc['version_creator'] = doc['original_creator']
remove_unwanted(doc)
update_word_count(doc)
set_item_expiry({}, doc)
if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_create([doc])
# Do the validation after Circular Reference check passes in Package Service
self.validate_embargo(doc)
if doc.get('media'):
self.mediaService.on_create([doc])
# let client create version 0 docs
if doc.get('version') == 0:
doc[config.VERSION] = doc['version']
if not doc.get('ingest_provider'):
doc['source'] = DEFAULT_SOURCE_VALUE_FOR_MANUAL_ARTICLES
def on_created(self, docs):
packages = [
doc for doc in docs if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE
]
if packages:
self.packageService.on_created(packages)
for doc in docs:
subject = get_subject(doc)
if subject:
msg = 'added new {{ type }} item about "{{ subject }}"'
else:
msg = 'added new {{ type }} item with empty header/title'
add_activity(ACTIVITY_CREATE,
msg,
self.datasource,
item=doc,
type=doc[ITEM_TYPE],
subject=subject)
push_content_notification(docs)
def on_update(self, updates, original):
updates[ITEM_OPERATION] = ITEM_UPDATE
is_update_allowed(original)
user = get_user()
if 'publish_schedule' in updates and original['state'] == 'scheduled':
# this is an deschedule action
self.deschedule_item(updates, original)
# check if there is a takes package and deschedule the takes package.
package = TakesPackageService().get_take_package(original)
if package and package.get('state') == 'scheduled':
package_updates = {
'publish_schedule': None,
'groups': package.get('groups')
}
self.patch(package.get(config.ID_FIELD), package_updates)
return
if updates.get('publish_schedule'):
if datetime.datetime.fromtimestamp(0).date() == updates.get(
'publish_schedule').date():
# publish_schedule field will be cleared
updates['publish_schedule'] = None
else:
# validate the schedule
if is_item_in_package(original):
raise SuperdeskApiError.\
badRequestError(message='This item is in a package' +
' it needs to be removed before the item can be scheduled!')
package = TakesPackageService().get_take_package(
original) or {}
validate_schedule(updates.get('publish_schedule'),
package.get(SEQUENCE, 1))
if 'unique_name' in updates and not is_admin(user) \
and (user['active_privileges'].get('metadata_uniquename', 0) == 0):
raise SuperdeskApiError.forbiddenError(
"Unauthorized to modify Unique Name")
remove_unwanted(updates)
if self.__is_req_for_save(updates):
update_state(original, updates)
lock_user = original.get('lock_user', None)
force_unlock = updates.get('force_unlock', False)
updates.setdefault('original_creator',
original.get('original_creator'))
str_user_id = str(user.get('_id')) if user else None
if lock_user and str(lock_user) != str_user_id and not force_unlock:
raise SuperdeskApiError.forbiddenError(
'The item was locked by another user')
updates['versioncreated'] = utcnow()
set_item_expiry(updates, original)
updates['version_creator'] = str_user_id
set_sign_off(updates, original=original)
update_word_count(updates)
if force_unlock:
del updates['force_unlock']
# create crops
crop_service = ArchiveCropService()
crop_service.validate_multiple_crops(updates, original)
crop_service.create_multiple_crops(updates, original)
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_update(updates, original)
update_version(updates, original)
# Do the validation after Circular Reference check passes in Package Service
updated = original.copy()
updated.update(updates)
self.validate_embargo(updated)
def on_updated(self, updates, original):
get_component(ItemAutosave).clear(original['_id'])
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_updated(updates, original)
ArchiveCropService().delete_replaced_crop_files(updates, original)
updated = copy(original)
updated.update(updates)
if config.VERSION in updates:
add_activity(
ACTIVITY_UPDATE,
'created new version {{ version }} for item {{ type }} about "{{ subject }}"',
self.datasource,
item=updated,
version=updates[config.VERSION],
subject=get_subject(updates, original),
type=updated[ITEM_TYPE])
push_content_notification([updated, original])
def on_replace(self, document, original):
document[ITEM_OPERATION] = ITEM_UPDATE
remove_unwanted(document)
user = get_user()
lock_user = original.get('lock_user', None)
force_unlock = document.get('force_unlock', False)
user_id = str(user.get('_id'))
if lock_user and str(lock_user) != user_id and not force_unlock:
raise SuperdeskApiError.forbiddenError(
'The item was locked by another user')
document['versioncreated'] = utcnow()
set_item_expiry(document, original)
document['version_creator'] = user_id
if force_unlock:
del document['force_unlock']
def on_replaced(self, document, original):
get_component(ItemAutosave).clear(original['_id'])
add_activity(ACTIVITY_UPDATE,
'replaced item {{ type }} about {{ subject }}',
self.datasource,
item=original,
type=original['type'],
subject=get_subject(original))
push_content_notification([document, original])
def on_deleted(self, doc):
if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_deleted(doc)
remove_media_files(doc)
add_activity(ACTIVITY_DELETE,
'removed item {{ type }} about {{ subject }}',
self.datasource,
item=doc,
type=doc[ITEM_TYPE],
subject=get_subject(doc))
push_content_notification([doc])
def replace(self, id, document, original):
return self.restore_version(id, document, original) or super().replace(
id, document, original)
def find_one(self, req, **lookup):
item = super().find_one(req, **lookup)
if item and str(item.get('task', {}).get('stage', '')) in \
get_resource_service('users').get_invisible_stages_ids(get_user().get('_id')):
raise SuperdeskApiError.forbiddenError(
"User does not have permissions to read the item.")
handle_existing_data(item)
return item
def restore_version(self, id, doc, original):
item_id = id
old_version = int(doc.get('old_version', 0))
last_version = int(doc.get('last_version', 0))
if (not all([item_id, old_version, last_version])):
return None
old = get_resource_service('archive_versions').find_one(
req=None, _id_document=item_id, _current_version=old_version)
if old is None:
raise SuperdeskApiError.notFoundError('Invalid version %s' %
old_version)
curr = get_resource_service(SOURCE).find_one(req=None, _id=item_id)
if curr is None:
raise SuperdeskApiError.notFoundError('Invalid item id %s' %
item_id)
if curr[config.VERSION] != last_version:
raise SuperdeskApiError.preconditionFailedError(
'Invalid last version %s' % last_version)
old['_id'] = old['_id_document']
old['_updated'] = old['versioncreated'] = utcnow()
set_item_expiry(old, doc)
del old['_id_document']
old[ITEM_OPERATION] = ITEM_RESTORE
resolve_document_version(old, SOURCE, 'PATCH', curr)
remove_unwanted(old)
set_sign_off(updates=old, original=curr)
super().replace(id=item_id, document=old, original=curr)
del doc['old_version']
del doc['last_version']
doc.update(old)
return item_id
def duplicate_content(self, original_doc):
"""
Duplicates the 'original_doc' including it's version history. Copy and Duplicate actions use this method.
:return: guid of the duplicated article
"""
if original_doc.get(ITEM_TYPE, '') == CONTENT_TYPE.COMPOSITE:
for groups in original_doc.get('groups'):
if groups.get('id') != 'root':
associations = groups.get('refs', [])
for assoc in associations:
if assoc.get(RESIDREF):
item, _item_id, _endpoint = self.packageService.get_associated_item(
assoc)
assoc[RESIDREF] = assoc[
'guid'] = self.duplicate_content(item)
return self._duplicate_item(original_doc)
def _duplicate_item(self, original_doc):
"""
Duplicates the 'original_doc' including it's version history. If the article being duplicated is contained
in a desk then the article state is changed to Submitted.
:return: guid of the duplicated article
"""
new_doc = original_doc.copy()
self._remove_after_copy(new_doc)
new_doc[ITEM_OPERATION] = ITEM_DUPLICATE
item_model = get_model(ItemModel)
on_duplicate_item(new_doc)
resolve_document_version(new_doc, SOURCE, 'PATCH', new_doc)
if original_doc.get('task', {}).get(
'desk') is not None and new_doc.get('state') != 'submitted':
new_doc[ITEM_STATE] = CONTENT_STATE.SUBMITTED
item_model.create([new_doc])
self._duplicate_versions(original_doc['guid'], new_doc)
return new_doc['guid']
def _remove_after_copy(self, copied_item):
"""
Removes the properties which doesn't make sense to have for an item after copy.
"""
del copied_item[config.ID_FIELD]
del copied_item['guid']
copied_item.pop(LINKED_IN_PACKAGES, None)
copied_item.pop(EMBARGO, None)
copied_item.pop('publish_schedule', None)
def _duplicate_versions(self, old_id, new_doc):
"""
Duplicates the version history of the article identified by old_id. Each version identifiers are changed
to have the identifiers of new_doc.
:param old_id: identifier to fetch version history
:param new_doc: identifiers from this doc will be used to create version history for the duplicated item.
"""
resource_def = app.config['DOMAIN']['archive']
version_id = versioned_id_field(resource_def)
old_versions = get_resource_service('archive_versions').get(
req=None, lookup={'guid': old_id})
new_versions = []
for old_version in old_versions:
old_version[version_id] = new_doc[config.ID_FIELD]
del old_version[config.ID_FIELD]
old_version['guid'] = new_doc['guid']
old_version['unique_name'] = new_doc['unique_name']
old_version['unique_id'] = new_doc['unique_id']
old_version['versioncreated'] = utcnow()
if old_version[VERSION] == new_doc[VERSION]:
old_version[ITEM_OPERATION] = new_doc[ITEM_OPERATION]
new_versions.append(old_version)
last_version = deepcopy(new_doc)
last_version['_id_document'] = new_doc['_id']
del last_version['_id']
new_versions.append(last_version)
if new_versions:
get_resource_service('archive_versions').post(new_versions)
def deschedule_item(self, updates, doc):
"""
Deschedule an item. This operation removed the item from publish queue and published collection.
:param dict updates: updates for the document
:param doc: original is document.
"""
updates['state'] = 'in_progress'
updates['publish_schedule'] = None
updates[ITEM_OPERATION] = ITEM_DESCHEDULE
# delete entries from publish queue
get_resource_service('publish_queue').delete_by_article_id(doc['_id'])
# delete entry from published repo
get_resource_service('published').delete_by_article_id(doc['_id'])
def validate_schedule(self, schedule):
if not isinstance(schedule, datetime.date):
raise SuperdeskApiError.badRequestError(
"Schedule date is not recognized")
if not schedule.date() or schedule.date().year <= 1970:
raise SuperdeskApiError.badRequestError(
"Schedule date is not recognized")
if not schedule.time():
raise SuperdeskApiError.badRequestError(
"Schedule time is not recognized")
if schedule < utcnow():
raise SuperdeskApiError.badRequestError(
"Schedule cannot be earlier than now")
def can_edit(self, item, user_id):
"""
Determines if the user can edit the item or not.
"""
# TODO: modify this function when read only permissions for stages are implemented
# TODO: and Content state related checking.
if not current_user_has_privilege('archive'):
return False, 'User does not have sufficient permissions.'
item_location = item.get('task')
if item_location:
if item_location.get('desk'):
if not superdesk.get_resource_service('user_desks').is_member(
user_id, item_location.get('desk')):
return False, 'User is not a member of the desk.'
elif item_location.get('user'):
if not str(item_location.get('user')) == str(user_id):
return False, 'Item belongs to another user.'
return True, ''
def remove_expired(self, doc):
"""
Removes the article from production if the state is spiked
"""
assert doc[ITEM_STATE] == CONTENT_STATE.SPIKED, \
"Article state is %s. Only Spiked Articles can be removed" % doc[ITEM_STATE]
doc_id = str(doc[config.ID_FIELD])
resource_def = app.config['DOMAIN']['archive_versions']
get_resource_service('archive_versions').delete(
lookup={versioned_id_field(resource_def): doc_id})
super().delete_action({config.ID_FIELD: doc_id})
def __is_req_for_save(self, doc):
"""
Patch of /api/archive is being used in multiple places. This method differentiates from the patch
triggered by user or not.
"""
if 'req_for_save' in doc:
req_for_save = doc['req_for_save']
del doc['req_for_save']
return req_for_save == 'true'
return True
def validate_embargo(self, item):
"""
Validates the embargo of the item. Following are checked:
1. Item can't be a package or a take or a re-write of another story
2. Publish Schedule and Embargo are mutually exclusive
3. Always a future date except in case of Corrected and Killed.
:raises: SuperdeskApiError.badRequestError() if the validation fails
"""
if item[ITEM_TYPE] != CONTENT_TYPE.COMPOSITE:
embargo = item.get(EMBARGO)
if embargo:
if item.get('publish_schedule'
) or item[ITEM_STATE] == CONTENT_STATE.SCHEDULED:
raise SuperdeskApiError.badRequestError(
"An item can't have both Publish Schedule and Embargo")
package = TakesPackageService().get_take_package(item)
if package:
raise SuperdeskApiError.badRequestError(
"Takes doesn't support Embargo")
if item.get('rewrite_of'):
raise SuperdeskApiError.badRequestError(
"Rewrites doesn't support Embargo")
if not isinstance(embargo,
datetime.date) or not embargo.time():
raise SuperdeskApiError.badRequestError("Invalid Embargo")
if item[ITEM_STATE] not in PUBLISH_STATES and embargo <= utcnow(
):
raise SuperdeskApiError.badRequestError(
"Embargo cannot be earlier than now")
elif item[
ITEM_TYPE] == CONTENT_TYPE.COMPOSITE and not self.takesService.is_takes_package(
item):
if item.get(EMBARGO):
raise SuperdeskApiError.badRequestError(
"A Package doesn't support Embargo")
self.packageService.check_if_any_item_in_package_has_embargo(item)
class ArchiveService(BaseService):
packageService = PackageService()
mediaService = ArchiveMediaService()
cropService = CropService()
def on_fetched(self, docs):
"""
Overriding this to handle existing data in Mongo & Elastic
"""
self.enhance_items(docs[config.ITEMS])
def on_fetched_item(self, doc):
self.enhance_items([doc])
def enhance_items(self, items):
for item in items:
handle_existing_data(item)
def on_create(self, docs):
on_create_item(docs)
for doc in docs:
if doc.get('body_footer') and is_normal_package(doc):
raise SuperdeskApiError.badRequestError(
"Package doesn't support Public Service Announcements")
self._test_readonly_stage(doc)
doc['version_creator'] = doc['original_creator']
remove_unwanted(doc)
update_word_count(doc)
set_item_expiry({}, doc)
if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_create([doc])
# Do the validation after Circular Reference check passes in Package Service
update_schedule_settings(doc, EMBARGO, doc.get(EMBARGO))
self.validate_embargo(doc)
update_associations(doc)
for assoc in doc.get(ASSOCIATIONS, {}).values():
self._set_association_timestamps(assoc, doc)
remove_unwanted(assoc)
if doc.get('media'):
self.mediaService.on_create([doc])
# let client create version 0 docs
if doc.get('version') == 0:
doc[config.VERSION] = doc['version']
self._add_desk_metadata(doc, {})
convert_task_attributes_to_objectId(doc)
def on_created(self, docs):
packages = [
doc for doc in docs if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE
]
if packages:
self.packageService.on_created(packages)
profiles = set()
for doc in docs:
subject = get_subject(doc)
if subject:
msg = 'added new {{ type }} item about "{{ subject }}"'
else:
msg = 'added new {{ type }} item with empty header/title'
add_activity(ACTIVITY_CREATE,
msg,
self.datasource,
item=doc,
type=doc[ITEM_TYPE],
subject=subject)
if doc.get('profile'):
profiles.add(doc['profile'])
self.cropService.update_media_references(doc, {})
if doc[ITEM_OPERATION] == ITEM_FETCH:
app.on_archive_item_updated({'task': doc.get('task')}, doc,
ITEM_FETCH)
else:
app.on_archive_item_updated({'task': doc.get('task')}, doc,
ITEM_CREATE)
get_resource_service('content_types').set_used(profiles)
push_content_notification(docs)
def on_update(self, updates, original):
"""Runs on archive update.
Overridden to validate the updates to the article and takes necessary actions depending on the updates.
In brief, it does the following:
1. Sets state, item operation, version created, version creator, sign off and word count.
2. Resets Item Expiry
3. Creates Crops if article is a picture
"""
user = get_user()
if ITEM_TYPE in updates:
del updates[ITEM_TYPE]
self._validate_updates(original, updates, user)
if self.__is_req_for_save(updates):
update_state(original, updates)
remove_unwanted(updates)
self._add_system_updates(original, updates, user)
self._add_desk_metadata(updates, original)
self._handle_media_updates(updates, original, user)
# send signal
superdesk.item_update.send(self, updates=updates, original=original)
def _handle_media_updates(self, updates, original, user):
update_associations(updates)
if original[ITEM_TYPE] == CONTENT_TYPE.PICTURE: # create crops
self.cropService.create_multiple_crops(updates, original)
if ASSOCIATIONS not in updates or not updates.get(ASSOCIATIONS):
return
body = updates.get("body_html", original.get("body_html", None))
# iterate over associations. Validate and process them if they are stored in database
for item_name, item_obj in updates.get(ASSOCIATIONS).items():
if not (item_obj and config.ID_FIELD in item_obj):
continue
item_id = item_obj[config.ID_FIELD]
media_item = {}
if app.settings.get('COPY_METADATA_FROM_PARENT') and item_obj.get(
ITEM_TYPE) in MEDIA_TYPES:
stored_item = (original.get(ASSOCIATIONS)
or {}).get(item_name) or item_obj
else:
media_item = stored_item = self.find_one(req=None, _id=item_id)
if not stored_item:
continue
self._validate_updates(stored_item, item_obj, user)
if stored_item[ITEM_TYPE] == CONTENT_TYPE.PICTURE: # create crops
CropService().create_multiple_crops(item_obj, stored_item)
if body and item_obj.get('description_text', None):
body = update_image_caption(body, item_name,
item_obj['description_text'])
# If the media item is not marked as 'used', mark it as used
if original.get(ITEM_TYPE) == CONTENT_TYPE.TEXT and \
(item_obj is not stored_item or not stored_item.get('used')):
if media_item is not stored_item:
media_item = self.find_one(req=None, _id=item_id)
if media_item and not media_item.get('used'):
self.system_update(media_item['_id'], {'used': True},
media_item)
stored_item['used'] = True
self._set_association_timestamps(item_obj, updates, new=False)
stored_item.update(item_obj)
updates[ASSOCIATIONS][item_name] = stored_item
if body:
updates["body_html"] = body
def on_updated(self, updates, original):
get_component(ItemAutosave).clear(original['_id'])
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_updated(updates, original)
updated = copy(original)
updated.update(updates)
if config.VERSION in updates:
add_activity(
ACTIVITY_UPDATE,
'created new version {{ version }} for item {{ type }} about "{{ subject }}"',
self.datasource,
item=updated,
version=updates[config.VERSION],
subject=get_subject(updates, original),
type=updated[ITEM_TYPE])
push_content_notification([updated, original])
get_resource_service('archive_broadcast').reset_broadcast_status(
updates, original)
if updates.get('profile'):
get_resource_service('content_types').set_used(
[updates.get('profile')])
self.cropService.update_media_references(updates, original)
def on_replace(self, document, original):
document[ITEM_OPERATION] = ITEM_UPDATE
remove_unwanted(document)
user = get_user()
lock_user = original.get('lock_user', None)
force_unlock = document.get('force_unlock', False)
user_id = str(user.get('_id'))
if lock_user and str(lock_user) != user_id and not force_unlock:
raise SuperdeskApiError.forbiddenError(
'The item was locked by another user')
document['versioncreated'] = utcnow()
set_item_expiry(document, original)
document['version_creator'] = user_id
if force_unlock:
del document['force_unlock']
def on_replaced(self, document, original):
get_component(ItemAutosave).clear(original['_id'])
add_activity(ACTIVITY_UPDATE,
'replaced item {{ type }} about {{ subject }}',
self.datasource,
item=original,
type=original['type'],
subject=get_subject(original))
push_content_notification([document, original])
self.cropService.update_media_references(document, original)
def on_deleted(self, doc):
get_component(ItemAutosave).clear(doc['_id'])
if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_deleted(doc)
remove_media_files(doc)
add_activity(ACTIVITY_DELETE,
'removed item {{ type }} about {{ subject }}',
self.datasource,
item=doc,
type=doc[ITEM_TYPE],
subject=get_subject(doc))
push_expired_notification([doc.get(config.ID_FIELD)])
app.on_archive_item_deleted(doc)
def replace(self, id, document, original):
return self.restore_version(id, document, original) or super().replace(
id, document, original)
def find_one(self, req, **lookup):
item = super().find_one(req, **lookup)
if item and str(item.get('task', {}).get('stage', '')) in \
get_resource_service('users').get_invisible_stages_ids(get_user().get('_id')):
raise SuperdeskApiError.forbiddenError(
"User does not have permissions to read the item.")
handle_existing_data(item)
return item
def restore_version(self, id, doc, original):
item_id = id
old_version = int(doc.get('old_version', 0))
last_version = int(doc.get('last_version', 0))
if (not all([item_id, old_version, last_version])):
return None
old = get_resource_service('archive_versions').find_one(
req=None, _id_document=item_id, _current_version=old_version)
if old is None:
raise SuperdeskApiError.notFoundError('Invalid version %s' %
old_version)
curr = get_resource_service(SOURCE).find_one(req=None, _id=item_id)
if curr is None:
raise SuperdeskApiError.notFoundError('Invalid item id %s' %
item_id)
if curr[config.VERSION] != last_version:
raise SuperdeskApiError.preconditionFailedError(
'Invalid last version %s' % last_version)
old['_id'] = old['_id_document']
old['_updated'] = old['versioncreated'] = utcnow()
set_item_expiry(old, doc)
old.pop('_id_document', None)
old.pop(SIGN_OFF, None)
old[ITEM_OPERATION] = ITEM_RESTORE
resolve_document_version(old, SOURCE, 'PATCH', curr)
remove_unwanted(old)
set_sign_off(updates=old, original=curr)
super().replace(id=item_id, document=old, original=curr)
old.pop('old_version', None)
old.pop('last_version', None)
doc.update(old)
return item_id
def duplicate_content(self, original_doc, state=None, extra_fields=None):
"""
Duplicates the 'original_doc' including it's version history. Copy and Duplicate actions use this method.
:return: guid of the duplicated article
"""
if original_doc.get(ITEM_TYPE, '') == CONTENT_TYPE.COMPOSITE:
for groups in original_doc.get('groups'):
if groups.get('id') != 'root':
associations = groups.get('refs', [])
for assoc in associations:
if assoc.get(RESIDREF):
item, _item_id, _endpoint = self.packageService.get_associated_item(
assoc)
assoc[RESIDREF] = assoc[
'guid'] = self.duplicate_content(item)
return self.duplicate_item(original_doc, state, extra_fields)
def duplicate_item(self,
original_doc,
state=None,
extra_fields=None,
operation=None):
"""Duplicates an item.
Duplicates the 'original_doc' including it's version history. If the article being duplicated is contained
in a desk then the article state is changed to Submitted.
:return: guid of the duplicated article
"""
new_doc = original_doc.copy()
self.remove_after_copy(new_doc, extra_fields)
on_duplicate_item(new_doc, original_doc, operation)
resolve_document_version(new_doc, SOURCE, 'PATCH', new_doc)
if original_doc.get('task',
{}).get('desk') is not None and new_doc.get(
ITEM_STATE) != CONTENT_STATE.SUBMITTED:
new_doc[ITEM_STATE] = CONTENT_STATE.SUBMITTED
if state:
new_doc[ITEM_STATE] = state
convert_task_attributes_to_objectId(new_doc)
get_model(ItemModel).create([new_doc])
self._duplicate_versions(original_doc['_id'], new_doc)
self._duplicate_history(original_doc['_id'], new_doc)
app.on_archive_item_updated({'duplicate_id': new_doc['guid']},
original_doc, operation or ITEM_DUPLICATE)
app.on_archive_item_updated({'duplicate_id': original_doc['_id']},
new_doc, operation or ITEM_DUPLICATED_FROM)
return new_doc['guid']
def remove_after_copy(self,
copied_item,
extra_fields=None,
delete_keys=None):
"""Removes the properties which doesn't make sense to have for an item after copy.
:param copied_item: item to copy
:param extra_fields: extra fields to copy besides content fields
"""
# get the archive schema keys
archive_schema_keys = list(
app.config['DOMAIN'][SOURCE]['schema'].keys())
archive_schema_keys.extend([
config.ID_FIELD, config.LAST_UPDATED, config.DATE_CREATED,
config.VERSION, config.ETAG
])
# Delete the keys that are not part of archive schema.
keys_to_delete = [
key for key in copied_item.keys() if key not in archive_schema_keys
]
keys_to_delete.extend([
config.ID_FIELD, 'guid', LINKED_IN_PACKAGES, EMBARGO,
PUBLISH_SCHEDULE, SCHEDULE_SETTINGS, 'lock_time', 'lock_action',
'lock_session', 'lock_user', SIGN_OFF, 'rewritten_by',
'rewrite_of', 'rewrite_sequence', 'highlights', 'marked_desks',
'_type', 'event_id', 'assignment_id', PROCESSED_FROM
])
if delete_keys:
keys_to_delete.extend(delete_keys)
if extra_fields:
keys_to_delete = [
key for key in keys_to_delete if key not in extra_fields
]
for key in keys_to_delete:
copied_item.pop(key, None)
# Copy should not preseve the SMS flag
if copied_item.get('flags', {}).get('marked_for_sms', False):
copied_item['flags']['marked_for_sms'] = False
task = copied_item.get('task', {})
task.pop(LAST_PRODUCTION_DESK, None)
task.pop(LAST_AUTHORING_DESK, None)
def _duplicate_versions(self, old_id, new_doc):
"""Duplicates versions for an item.
Duplicates the versions of the article identified by old_id. Each version identifiers are changed
to have the identifiers of new_doc.
:param old_id: identifier to fetch versions
:param new_doc: identifiers from this doc will be used to create versions for the duplicated item.
"""
resource_def = app.config['DOMAIN']['archive']
version_id = versioned_id_field(resource_def)
old_versions = get_resource_service('archive_versions').get(
req=None, lookup={version_id: old_id})
new_versions = []
for old_version in old_versions:
old_version[version_id] = new_doc[config.ID_FIELD]
del old_version[config.ID_FIELD]
old_version['guid'] = new_doc['guid']
old_version['unique_name'] = new_doc['unique_name']
old_version['unique_id'] = new_doc['unique_id']
old_version['versioncreated'] = utcnow()
if old_version[config.VERSION] == new_doc[config.VERSION]:
old_version[ITEM_OPERATION] = new_doc[ITEM_OPERATION]
new_versions.append(old_version)
last_version = deepcopy(new_doc)
last_version['_id_document'] = new_doc['_id']
del last_version['_id']
new_versions.append(last_version)
if new_versions:
get_resource_service('archive_versions').post(new_versions)
def _duplicate_history(self, old_id, new_doc):
"""Duplicates history for an item.
Duplicates the history of the article identified by old_id. Each history identifiers are changed
to have the identifiers of new_doc.
:param old_id: identifier to fetch history
:param new_doc: identifiers from this doc will be used to create version history for the duplicated item.
"""
old_history_items = get_resource_service('archive_history').get(
req=None, lookup={'item_id': old_id})
new_history_items = []
for old_history_item in old_history_items:
del old_history_item[config.ID_FIELD]
old_history_item['item_id'] = new_doc['guid']
if not old_history_item.get('original_item_id'):
old_history_item['original_item_id'] = old_id
new_history_items.append(old_history_item)
if new_history_items:
get_resource_service('archive_history').post(new_history_items)
def update(self, id, updates, original):
if updates.get(ASSOCIATIONS):
for association in updates[ASSOCIATIONS].values():
self._set_association_timestamps(association,
updates,
new=False)
remove_unwanted(association)
# this needs to here as resolve_nested_documents (in eve) will add the schedule_settings
if PUBLISH_SCHEDULE in updates and original[
ITEM_STATE] == CONTENT_STATE.SCHEDULED:
self.deschedule_item(updates,
original) # this is an deschedule action
return super().update(id, updates, original)
def deschedule_item(self, updates, original):
"""Deschedule an item.
This operation removed the item from publish queue and published collection.
:param dict updates: updates for the document
:param original: original is document.
"""
updates[ITEM_STATE] = CONTENT_STATE.PROGRESS
updates[PUBLISH_SCHEDULE] = original[PUBLISH_SCHEDULE]
updates[SCHEDULE_SETTINGS] = original[SCHEDULE_SETTINGS]
updates[ITEM_OPERATION] = ITEM_DESCHEDULE
# delete entry from published repo
get_resource_service('published').delete_by_article_id(original['_id'])
def can_edit(self, item, user_id):
"""
Determines if the user can edit the item or not.
"""
# TODO: modify this function when read only permissions for stages are implemented
# TODO: and Content state related checking.
if not current_user_has_privilege('archive'):
return False, 'User does not have sufficient permissions.'
item_location = item.get('task')
if item_location:
if item_location.get('desk'):
if not superdesk.get_resource_service('user_desks').is_member(
user_id, item_location.get('desk')):
return False, 'User is not a member of the desk.'
elif item_location.get('user'):
if not str(item_location.get('user')) == str(user_id):
return False, 'Item belongs to another user.'
return True, ''
def delete_by_article_ids(self, ids):
"""Remove the content
:param list ids: list of ids to be removed
"""
version_field = versioned_id_field(
app.config['DOMAIN']['archive_versions'])
get_resource_service('archive_versions').delete_action(
lookup={version_field: {
'$in': ids
}})
super().delete_action({config.ID_FIELD: {'$in': ids}})
def _set_association_timestamps(self, assoc_item, updates, new=True):
if type(assoc_item) == dict:
assoc_item[config.LAST_UPDATED] = updates.get(
config.LAST_UPDATED, datetime.datetime.now())
if new:
assoc_item[config.DATE_CREATED] = datetime.datetime.now()
elif config.DATE_CREATED in assoc_item:
del assoc_item[config.DATE_CREATED]
def __is_req_for_save(self, doc):
"""Checks if doc contains req_for_save key.
Patch of /api/archive is being used in multiple places. This method differentiates from the patch
triggered by user or not.
:param dictionary doc: doc to test
"""
if 'req_for_save' in doc:
req_for_save = doc['req_for_save']
del doc['req_for_save']
return req_for_save == 'true'
return True
def validate_embargo(self, item):
"""Validates the embargo of the item.
Following are checked:
1. Item can't be a package or a re-write of another story
2. Publish Schedule and Embargo are mutually exclusive
3. Always a future date except in case of Corrected and Killed.
:raises: SuperdeskApiError.badRequestError() if the validation fails
"""
if item[ITEM_TYPE] != CONTENT_TYPE.COMPOSITE:
if EMBARGO in item:
embargo = item.get(SCHEDULE_SETTINGS,
{}).get('utc_{}'.format(EMBARGO))
if embargo:
if item.get(PUBLISH_SCHEDULE) or item[
ITEM_STATE] == CONTENT_STATE.SCHEDULED:
raise SuperdeskApiError.badRequestError(
"An item can't have both Publish Schedule and Embargo"
)
if (item[ITEM_STATE] not in {
CONTENT_STATE.KILLED, CONTENT_STATE.RECALLED, CONTENT_STATE.SCHEDULED}) \
and embargo <= utcnow():
raise SuperdeskApiError.badRequestError(
"Embargo cannot be earlier than now")
if item.get('rewrite_of'):
raise SuperdeskApiError.badRequestError(
"Rewrites doesn't support Embargo")
if not isinstance(embargo,
datetime.date) or not embargo.time():
raise SuperdeskApiError.badRequestError(
"Invalid Embargo")
elif is_normal_package(item):
if item.get(EMBARGO):
raise SuperdeskApiError.badRequestError(
"A Package doesn't support Embargo")
self.packageService.check_if_any_item_in_package_has_embargo(item)
def _test_readonly_stage(self, item, updates=None):
"""If item is created or updated on readonly stage abort it.
:param item: edited/new item
:param updates: item updates
"""
def abort_if_readonly_stage(stage_id):
stage = superdesk.get_resource_service('stages').find_one(
req=None, _id=stage_id)
if stage.get('local_readonly'):
flask.abort(403, response={'readonly': True})
orig_stage_id = item.get('task', {}).get('stage')
if orig_stage_id and get_user() and not item.get(INGEST_ID):
abort_if_readonly_stage(orig_stage_id)
if updates:
dest_stage_id = updates.get('task', {}).get('stage')
if dest_stage_id and get_user() and not item.get(INGEST_ID):
abort_if_readonly_stage(dest_stage_id)
def _validate_updates(self, original, updates, user):
"""Validates updates to the article for the below conditions.
If any of these conditions are met then exception is raised:
1. Is article locked by another user other than the user requesting for update
2. Is state of the article is Killed or Recalled?
3. Is user trying to update the package with Public Service Announcements?
4. Is user authorized to update unique name of the article?
5. Is user trying to update the genre of a broadcast article?
6. Is article being scheduled and is in a package?
7. Is article being scheduled and schedule timestamp is invalid?
8. Does article has valid crops if the article type is a picture?
9. Is article a valid package if the article type is a package?
10. Does article has a valid Embargo?
11. Make sure that there are no duplicate anpa_category codes in the article.
12. Make sure there are no duplicate subjects in the upadte
13. Item is on readonly stage.
:raises:
SuperdeskApiError.forbiddenError()
- if state of the article is killed or user is not authorized to update unique name or if article is
locked by another user
SuperdeskApiError.badRequestError()
- if Public Service Announcements are being added to a package or genre is being updated for a
broadcast, is invalid for scheduling, the updates contain duplicate anpa_category or subject codes
"""
updated = original.copy()
updated.update(updates)
self._test_readonly_stage(original, updates)
lock_user = original.get('lock_user', None)
force_unlock = updates.get('force_unlock', False)
str_user_id = str(user.get(config.ID_FIELD)) if user else None
if lock_user and str(lock_user) != str_user_id and not force_unlock:
raise SuperdeskApiError.forbiddenError(
'The item was locked by another user')
if original.get(ITEM_STATE) in {
CONTENT_STATE.KILLED, CONTENT_STATE.RECALLED
}:
raise SuperdeskApiError.forbiddenError(
"Item isn't in a valid state to be updated.")
if updates.get('body_footer') and is_normal_package(original):
raise SuperdeskApiError.badRequestError(
"Package doesn't support Public Service Announcements")
if 'unique_name' in updates and not is_admin(user) \
and (user['active_privileges'].get('metadata_uniquename', 0) == 0) \
and not force_unlock:
raise SuperdeskApiError.forbiddenError(
"Unauthorized to modify Unique Name")
# if broadcast then update to genre is not allowed.
if original.get('broadcast') and updates.get('genre') and \
any(genre.get('qcode', '').lower() != BROADCAST_GENRE.lower() for genre in updates.get('genre')):
raise SuperdeskApiError.badRequestError(
'Cannot change the genre for broadcast content.')
if PUBLISH_SCHEDULE in updates or "schedule_settings" in updates:
if is_item_in_package(original) and not force_unlock:
raise SuperdeskApiError.badRequestError(
'This item is in a package and it needs to be removed before the item can be scheduled!'
)
update_schedule_settings(updated, PUBLISH_SCHEDULE,
updated.get(PUBLISH_SCHEDULE))
if updates.get(PUBLISH_SCHEDULE):
validate_schedule(
updated.get(SCHEDULE_SETTINGS,
{}).get('utc_{}'.format(PUBLISH_SCHEDULE)))
updates[SCHEDULE_SETTINGS] = updated.get(SCHEDULE_SETTINGS, {})
if original[ITEM_TYPE] == CONTENT_TYPE.PICTURE:
CropService().validate_multiple_crops(updates, original)
elif original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_update(updates, original)
# update the embargo date
update_schedule_settings(updated, EMBARGO, updated.get(EMBARGO))
# Do the validation after Circular Reference check passes in Package Service
self.validate_embargo(updated)
if EMBARGO in updates or "schedule_settings" in updates:
updates[SCHEDULE_SETTINGS] = updated.get(SCHEDULE_SETTINGS, {})
# Ensure that there are no duplicate categories in the update
category_qcodes = [
q['qcode'] for q in updates.get('anpa_category', []) or []
]
if category_qcodes and len(category_qcodes) != len(
set(category_qcodes)):
raise SuperdeskApiError.badRequestError(
"Duplicate category codes are not allowed")
# Ensure that there are no duplicate subjects in the update
subject_qcodes = [q['qcode'] for q in updates.get('subject', []) or []]
if subject_qcodes and len(subject_qcodes) != len(set(subject_qcodes)):
raise SuperdeskApiError.badRequestError(
"Duplicate subjects are not allowed")
def _add_system_updates(self, original, updates, user):
"""Adds system updates to item.
As the name suggests, this method adds properties which are derived based on updates sent in the request.
1. Sets item operation, version created, version creator, sign off and word count.
2. Resets Item Expiry
"""
convert_task_attributes_to_objectId(updates)
updates[ITEM_OPERATION] = ITEM_UPDATE
updates.setdefault('original_creator',
original.get('original_creator'))
updates['versioncreated'] = utcnow()
updates['version_creator'] = str(user.get(
config.ID_FIELD)) if user else None
update_word_count(updates, original)
update_version(updates, original)
set_item_expiry(updates, original)
set_sign_off(updates, original=original)
set_dateline(updates, original)
# Clear publish_schedule field
if updates.get(PUBLISH_SCHEDULE) \
and datetime.datetime.fromtimestamp(0).date() == updates.get(PUBLISH_SCHEDULE).date():
updates[PUBLISH_SCHEDULE] = None
updates[SCHEDULE_SETTINGS] = {}
if updates.get('force_unlock', False):
del updates['force_unlock']
def get_expired_items(self, expiry_datetime, invalid_only=False):
"""Get the expired items.
Where content state is not scheduled and the item matches given parameters
:param datetime expiry_datetime: expiry datetime
:param bool invalid_only: True only invalid items
:return pymongo.cursor: expired non published items.
"""
unique_id = 0
while True:
req = ParsedRequest()
req.sort = 'unique_id'
query = {
'$and': [{
'expiry': {
'$lte': date_to_str(expiry_datetime)
}
}, {
'$or': [{
'task.desk': {
'$ne': None
}
}, {
ITEM_STATE: CONTENT_STATE.SPIKED,
'task.desk': None
}]
}]
}
query['$and'].append({'unique_id': {'$gt': unique_id}})
if invalid_only:
query['$and'].append({'expiry_status': 'invalid'})
else:
query['$and'].append({'expiry_status': {'$ne': 'invalid'}})
req.where = json.dumps(query)
req.max_results = config.MAX_EXPIRY_QUERY_LIMIT
items = list(self.get_from_mongo(req=req, lookup=None))
if not len(items):
break
unique_id = items[-1]['unique_id']
yield items
def _add_desk_metadata(self, updates, original):
"""Populate updates metadata from item desk in case it's set.
It will only add data which is not set yet on the item.
:param updates: updates to item that should be saved
:param original: original item version before update
"""
return get_resource_service('desks').apply_desk_metadata(
updates, original)
class ArchiveService(BaseService):
packageService = PackageService()
takesService = TakesPackageService()
mediaService = ArchiveMediaService()
def on_fetched(self, docs):
"""
Overriding this to handle existing data in Mongo & Elastic
"""
self.__enhance_items(docs[config.ITEMS])
def on_fetched_item(self, doc):
self.__enhance_items([doc])
def __enhance_items(self, items):
for item in items:
handle_existing_data(item)
self.takesService.enhance_with_package_info(item)
def on_create(self, docs):
on_create_item(docs)
for doc in docs:
if doc.get('body_footer') and is_normal_package(doc):
raise SuperdeskApiError.badRequestError("Package doesn't support Public Service Announcements")
doc['version_creator'] = doc['original_creator']
remove_unwanted(doc)
update_word_count(doc)
set_item_expiry({}, doc)
if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_create([doc])
# Do the validation after Circular Reference check passes in Package Service
self.validate_embargo(doc)
if doc.get('media'):
self.mediaService.on_create([doc])
# let client create version 0 docs
if doc.get('version') == 0:
doc[config.VERSION] = doc['version']
if not doc.get('ingest_provider'):
doc['source'] = DEFAULT_SOURCE_VALUE_FOR_MANUAL_ARTICLES
doc.setdefault('priority', DEFAULT_PRIORITY_VALUE_FOR_MANUAL_ARTICLES)
doc.setdefault('urgency', DEFAULT_URGENCY_VALUE_FOR_MANUAL_ARTICLES)
convert_task_attributes_to_objectId(doc)
def on_created(self, docs):
packages = [doc for doc in docs if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE]
if packages:
self.packageService.on_created(packages)
for doc in docs:
subject = get_subject(doc)
if subject:
msg = 'added new {{ type }} item about "{{ subject }}"'
else:
msg = 'added new {{ type }} item with empty header/title'
add_activity(ACTIVITY_CREATE, msg,
self.datasource, item=doc, type=doc[ITEM_TYPE], subject=subject)
push_content_notification(docs)
def on_update(self, updates, original):
"""
Overridden to validate the updates to the article and take necessary actions depending on the updates. In brief,
it does the following:
1. Sets state, item operation, version created, version creator, sign off and word count.
2. Resets Item Expiry
3. If the request is to de-schedule then checks and de-schedules the associated Takes Package also.
4. Creates Crops if article is a picture
"""
user = get_user()
self._validate_updates(original, updates, user)
if 'publish_schedule' in updates and original[ITEM_STATE] == CONTENT_STATE.SCHEDULED:
self.deschedule_item(updates, original) # this is an deschedule action
# check if there is a takes package and deschedule the takes package.
package = TakesPackageService().get_take_package(original)
if package and package.get('state') == 'scheduled':
package_updates = {'publish_schedule': None, 'groups': package.get('groups')}
self.patch(package.get(config.ID_FIELD), package_updates)
return
if self.__is_req_for_save(updates):
update_state(original, updates)
remove_unwanted(updates)
self._add_system_updates(original, updates, user)
if original[ITEM_TYPE] == CONTENT_TYPE.PICTURE: # create crops
CropService().create_multiple_crops(updates, original)
def on_updated(self, updates, original):
get_component(ItemAutosave).clear(original['_id'])
if original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_updated(updates, original)
CropService().delete_replaced_crop_files(updates, original)
updated = copy(original)
updated.update(updates)
if config.VERSION in updates:
add_activity(ACTIVITY_UPDATE, 'created new version {{ version }} for item {{ type }} about "{{ subject }}"',
self.datasource, item=updated,
version=updates[config.VERSION], subject=get_subject(updates, original),
type=updated[ITEM_TYPE])
push_content_notification([updated, original])
get_resource_service('archive_broadcast').reset_broadcast_status(updates, original)
def on_replace(self, document, original):
document[ITEM_OPERATION] = ITEM_UPDATE
remove_unwanted(document)
user = get_user()
lock_user = original.get('lock_user', None)
force_unlock = document.get('force_unlock', False)
user_id = str(user.get('_id'))
if lock_user and str(lock_user) != user_id and not force_unlock:
raise SuperdeskApiError.forbiddenError('The item was locked by another user')
document['versioncreated'] = utcnow()
set_item_expiry(document, original)
document['version_creator'] = user_id
if force_unlock:
del document['force_unlock']
def on_replaced(self, document, original):
get_component(ItemAutosave).clear(original['_id'])
add_activity(ACTIVITY_UPDATE, 'replaced item {{ type }} about {{ subject }}',
self.datasource, item=original,
type=original['type'], subject=get_subject(original))
push_content_notification([document, original])
def on_deleted(self, doc):
if doc[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_deleted(doc)
remove_media_files(doc)
add_activity(ACTIVITY_DELETE, 'removed item {{ type }} about {{ subject }}',
self.datasource, item=doc,
type=doc[ITEM_TYPE], subject=get_subject(doc))
push_content_notification([doc])
def replace(self, id, document, original):
return self.restore_version(id, document, original) or super().replace(id, document, original)
def find_one(self, req, **lookup):
item = super().find_one(req, **lookup)
if item and str(item.get('task', {}).get('stage', '')) in \
get_resource_service('users').get_invisible_stages_ids(get_user().get('_id')):
raise SuperdeskApiError.forbiddenError("User does not have permissions to read the item.")
handle_existing_data(item)
return item
def restore_version(self, id, doc, original):
item_id = id
old_version = int(doc.get('old_version', 0))
last_version = int(doc.get('last_version', 0))
if (not all([item_id, old_version, last_version])):
return None
old = get_resource_service('archive_versions').find_one(req=None, _id_document=item_id,
_current_version=old_version)
if old is None:
raise SuperdeskApiError.notFoundError('Invalid version %s' % old_version)
curr = get_resource_service(SOURCE).find_one(req=None, _id=item_id)
if curr is None:
raise SuperdeskApiError.notFoundError('Invalid item id %s' % item_id)
if curr[config.VERSION] != last_version:
raise SuperdeskApiError.preconditionFailedError('Invalid last version %s' % last_version)
old['_id'] = old['_id_document']
old['_updated'] = old['versioncreated'] = utcnow()
set_item_expiry(old, doc)
del old['_id_document']
old[ITEM_OPERATION] = ITEM_RESTORE
resolve_document_version(old, SOURCE, 'PATCH', curr)
remove_unwanted(old)
set_sign_off(updates=old, original=curr)
super().replace(id=item_id, document=old, original=curr)
del doc['old_version']
del doc['last_version']
doc.update(old)
return item_id
def duplicate_content(self, original_doc):
"""
Duplicates the 'original_doc' including it's version history. Copy and Duplicate actions use this method.
:return: guid of the duplicated article
"""
if original_doc.get(ITEM_TYPE, '') == CONTENT_TYPE.COMPOSITE:
for groups in original_doc.get('groups'):
if groups.get('id') != 'root':
associations = groups.get('refs', [])
for assoc in associations:
if assoc.get(RESIDREF):
item, _item_id, _endpoint = self.packageService.get_associated_item(assoc)
assoc[RESIDREF] = assoc['guid'] = self.duplicate_content(item)
return self._duplicate_item(original_doc)
def _duplicate_item(self, original_doc):
"""
Duplicates the 'original_doc' including it's version history. If the article being duplicated is contained
in a desk then the article state is changed to Submitted.
:return: guid of the duplicated article
"""
new_doc = original_doc.copy()
self._remove_after_copy(new_doc)
on_duplicate_item(new_doc)
resolve_document_version(new_doc, SOURCE, 'PATCH', new_doc)
if original_doc.get('task', {}).get('desk') is not None and new_doc.get('state') != 'submitted':
new_doc[ITEM_STATE] = CONTENT_STATE.SUBMITTED
convert_task_attributes_to_objectId(new_doc)
get_model(ItemModel).create([new_doc])
self._duplicate_versions(original_doc['guid'], new_doc)
return new_doc['guid']
def _remove_after_copy(self, copied_item):
"""
Removes the properties which doesn't make sense to have for an item after copy.
"""
del copied_item[config.ID_FIELD]
del copied_item['guid']
copied_item.pop(LINKED_IN_PACKAGES, None)
copied_item.pop(EMBARGO, None)
copied_item.pop('publish_schedule', None)
copied_item.pop('lock_time', None)
copied_item.pop('lock_session', None)
copied_item.pop('lock_user', None)
task = copied_item.get('task', {})
task.pop(LAST_PRODUCTION_DESK, None)
task.pop(LAST_AUTHORING_DESK, None)
def _duplicate_versions(self, old_id, new_doc):
"""
Duplicates the version history of the article identified by old_id. Each version identifiers are changed
to have the identifiers of new_doc.
:param old_id: identifier to fetch version history
:param new_doc: identifiers from this doc will be used to create version history for the duplicated item.
"""
resource_def = app.config['DOMAIN']['archive']
version_id = versioned_id_field(resource_def)
old_versions = get_resource_service('archive_versions').get(req=None, lookup={'guid': old_id})
new_versions = []
for old_version in old_versions:
old_version[version_id] = new_doc[config.ID_FIELD]
del old_version[config.ID_FIELD]
old_version['guid'] = new_doc['guid']
old_version['unique_name'] = new_doc['unique_name']
old_version['unique_id'] = new_doc['unique_id']
old_version['versioncreated'] = utcnow()
if old_version[VERSION] == new_doc[VERSION]:
old_version[ITEM_OPERATION] = new_doc[ITEM_OPERATION]
new_versions.append(old_version)
last_version = deepcopy(new_doc)
last_version['_id_document'] = new_doc['_id']
del last_version['_id']
new_versions.append(last_version)
if new_versions:
get_resource_service('archive_versions').post(new_versions)
def deschedule_item(self, updates, doc):
"""
Deschedule an item. This operation removed the item from publish queue and published collection.
:param dict updates: updates for the document
:param doc: original is document.
"""
updates['state'] = 'in_progress'
updates['publish_schedule'] = None
updates[ITEM_OPERATION] = ITEM_DESCHEDULE
# delete entry from published repo
get_resource_service('published').delete_by_article_id(doc['_id'])
def validate_schedule(self, schedule):
if not isinstance(schedule, datetime.date):
raise SuperdeskApiError.badRequestError("Schedule date is not recognized")
if not schedule.date() or schedule.date().year <= 1970:
raise SuperdeskApiError.badRequestError("Schedule date is not recognized")
if not schedule.time():
raise SuperdeskApiError.badRequestError("Schedule time is not recognized")
if schedule < utcnow():
raise SuperdeskApiError.badRequestError("Schedule cannot be earlier than now")
def can_edit(self, item, user_id):
"""
Determines if the user can edit the item or not.
"""
# TODO: modify this function when read only permissions for stages are implemented
# TODO: and Content state related checking.
if not current_user_has_privilege('archive'):
return False, 'User does not have sufficient permissions.'
item_location = item.get('task')
if item_location:
if item_location.get('desk'):
if not superdesk.get_resource_service('user_desks').is_member(user_id, item_location.get('desk')):
return False, 'User is not a member of the desk.'
elif item_location.get('user'):
if not str(item_location.get('user')) == str(user_id):
return False, 'Item belongs to another user.'
return True, ''
def delete_by_article_ids(self, ids):
"""
remove the content
:param list ids: list of ids to be removed
"""
version_field = versioned_id_field(app.config['DOMAIN']['archive_versions'])
get_resource_service('archive_versions').delete(lookup={version_field: {'$in': ids}})
super().delete_action({config.ID_FIELD: {'$in': ids}})
def __is_req_for_save(self, doc):
"""
Patch of /api/archive is being used in multiple places. This method differentiates from the patch
triggered by user or not.
"""
if 'req_for_save' in doc:
req_for_save = doc['req_for_save']
del doc['req_for_save']
return req_for_save == 'true'
return True
def validate_embargo(self, item):
"""
Validates the embargo of the item. Following are checked:
1. Item can't be a package or a take or a re-write of another story
2. Publish Schedule and Embargo are mutually exclusive
3. Always a future date except in case of Corrected and Killed.
:raises: SuperdeskApiError.badRequestError() if the validation fails
"""
if item[ITEM_TYPE] != CONTENT_TYPE.COMPOSITE:
embargo = item.get(EMBARGO)
if embargo:
if item.get('publish_schedule') or item[ITEM_STATE] == CONTENT_STATE.SCHEDULED:
raise SuperdeskApiError.badRequestError("An item can't have both Publish Schedule and Embargo")
package = TakesPackageService().get_take_package(item)
if package:
raise SuperdeskApiError.badRequestError("Takes doesn't support Embargo")
if item.get('rewrite_of'):
raise SuperdeskApiError.badRequestError("Rewrites doesn't support Embargo")
if not isinstance(embargo, datetime.date) or not embargo.time():
raise SuperdeskApiError.badRequestError("Invalid Embargo")
if item[ITEM_STATE] not in PUBLISH_STATES and embargo <= utcnow():
raise SuperdeskApiError.badRequestError("Embargo cannot be earlier than now")
elif is_normal_package(item):
if item.get(EMBARGO):
raise SuperdeskApiError.badRequestError("A Package doesn't support Embargo")
self.packageService.check_if_any_item_in_package_has_embargo(item)
def _validate_updates(self, original, updates, user):
"""
Validates updates to the article for the below conditions, if any of them then exception is raised:
1. Is article locked by another user other than the user requesting for update
2. Is state of the article is Killed?
3. Is user trying to update the package with Public Service Announcements?
4. Is user authorized to update unique name of the article?
5. Is user trying to update the genre of a broadcast article?
6. Is article being scheduled and is in a package?
7. Is article being scheduled and schedule timestamp is invalid?
8. Does article has valid crops if the article type is a picture?
9. Is article a valid package if the article type is a package?
10. Does article has a valid Embargo?
11. Make sure that there are no duplicate anpa_category codes in the article.
12. Make sure there are no duplicate subjects in the upadte
:raises:
SuperdeskApiError.forbiddenError()
- if state of the article is killed or user is not authorized to update unique name or if article is
locked by another user
SuperdeskApiError.badRequestError()
- if Public Service Announcements are being added to a package or genre is being updated for a
broadcast, is invalid for scheduling, the updates contain duplicate anpa_category or subject codes
"""
lock_user = original.get('lock_user', None)
force_unlock = updates.get('force_unlock', False)
str_user_id = str(user.get(config.ID_FIELD)) if user else None
if lock_user and str(lock_user) != str_user_id and not force_unlock:
raise SuperdeskApiError.forbiddenError('The item was locked by another user')
if original.get(ITEM_STATE) == CONTENT_STATE.KILLED:
raise SuperdeskApiError.forbiddenError("Item isn't in a valid state to be updated.")
if updates.get('body_footer') and is_normal_package(original):
raise SuperdeskApiError.badRequestError("Package doesn't support Public Service Announcements")
if 'unique_name' in updates and not is_admin(user) \
and (user['active_privileges'].get('metadata_uniquename', 0) == 0):
raise SuperdeskApiError.forbiddenError("Unauthorized to modify Unique Name")
# if broadcast then update to genre is not allowed.
if original.get('broadcast') and updates.get('genre') and \
any(genre.get('value', '').lower() != BROADCAST_GENRE.lower() for genre in updates.get('genre')):
raise SuperdeskApiError.badRequestError('Cannot change the genre for broadcast content.')
if updates.get('publish_schedule') and original[ITEM_STATE] != CONTENT_STATE.SCHEDULED \
and datetime.datetime.fromtimestamp(0).date() != updates['publish_schedule'].date():
if is_item_in_package(original):
raise SuperdeskApiError.badRequestError(
'This item is in a package and it needs to be removed before the item can be scheduled!')
package = TakesPackageService().get_take_package(original) or {}
validate_schedule(updates['publish_schedule'], package.get(SEQUENCE, 1))
if original[ITEM_TYPE] == CONTENT_TYPE.PICTURE:
CropService().validate_multiple_crops(updates, original)
elif original[ITEM_TYPE] == CONTENT_TYPE.COMPOSITE:
self.packageService.on_update(updates, original)
# Do the validation after Circular Reference check passes in Package Service
updated = original.copy()
updated.update(updates)
self.validate_embargo(updated)
# Ensure that there are no duplicate categories in the update
category_qcodes = [q['qcode'] for q in updates.get('anpa_category', []) or []]
if category_qcodes and len(category_qcodes) != len(set(category_qcodes)):
raise SuperdeskApiError.badRequestError("Duplicate category codes are not allowed")
# Ensure that there are no duplicate subjects in the update
subject_qcodes = [q['qcode'] for q in updates.get('subject', []) or []]
if subject_qcodes and len(subject_qcodes) != len(set(subject_qcodes)):
raise SuperdeskApiError.badRequestError("Duplicate subjects are not allowed")
def _add_system_updates(self, original, updates, user):
"""
As the name suggests, this method adds properties which are derived based on updates sent in the request.
1. Sets item operation, version created, version creator, sign off and word count.
2. Resets Item Expiry
"""
convert_task_attributes_to_objectId(updates)
updates[ITEM_OPERATION] = ITEM_UPDATE
updates.setdefault('original_creator', original.get('original_creator'))
updates['versioncreated'] = utcnow()
updates['version_creator'] = str(user.get(config.ID_FIELD)) if user else None
update_word_count(updates)
update_version(updates, original)
set_item_expiry(updates, original)
set_sign_off(updates, original=original)
# Clear publish_schedule field
if updates.get('publish_schedule') \
and datetime.datetime.fromtimestamp(0).date() == updates.get('publish_schedule').date():
updates['publish_schedule'] = None
if updates.get('force_unlock', False):
del updates['force_unlock']
def get_expired_items(self, expiry_datetime):
"""
Get the expired items where content state is not scheduled
and
:param datetime expiry_datetime: expiry datetime
:return pymongo.cursor: expired non published items.
"""
query = {
'$and': [
{'expiry': {'$lte': date_to_str(expiry_datetime)}},
{'$or': [
{'task.desk': {'$ne': None}},
{ITEM_STATE: CONTENT_STATE.SPIKED, 'task.desk': None}
]}
]
}
req = ParsedRequest()
req.max_results = config.MAX_EXPIRY_QUERY_LIMIT
req.sort = 'expiry,_created'
return self.get_from_mongo(req=None, lookup=query)
def _can_remove_item(self,
item,
processed_item=None,
preserve_published_desks=None):
"""Recursively checks if the item can be removed.
:param dict item: item to be remove
:param set processed_item: processed items
:return: True if item can be removed, False otherwise.
"""
if processed_item is None:
processed_item = dict()
item_refs = []
package_service = PackageService()
archive_service = get_resource_service(ARCHIVE)
if item.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
# Get the item references for is package
item_refs = package_service.get_residrefs(item)
if item.get(ITEM_TYPE) in [
CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED
]:
broadcast_items = get_resource_service(
'archive_broadcast').get_broadcast_items_from_master_story(
item)
# If master story expires then check if broadcast item is included in a package.
# If included in a package then check the package expiry.
item_refs.extend([
broadcast_item.get(config.ID_FIELD)
for broadcast_item in broadcast_items
])
if item.get('rewrite_of'):
item_refs.append(item.get('rewrite_of'))
if item.get('rewritten_by'):
item_refs.append(item.get('rewritten_by'))
# get the list of associated item ids
if item.get(ITEM_TYPE) in MEDIA_TYPES:
item_refs.extend(self._get_associated_items(item))
# get item reference where this referred
item_refs.extend(package_service.get_linked_in_package_ids(item))
# check item refs in the ids to remove set
is_expired = item.get('expiry') and item.get('expiry') < utcnow()
# if the item is published or corrected and desk has preserve_published_content as true
if preserve_published_desks and \
item.get(ITEM_STATE) in {CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED} and \
item.get('task').get('desk') in preserve_published_desks:
is_expired = False
# If the item is associated with a planning assignment and not published then preserve it
if item.get('assignment_id') and item.get(
ITEM_STATE) not in PUBLISH_STATES:
try:
assignment = superdesk.get_resource_service(
'assignments').find_one(req=None,
_id=item['assignment_id'])
if assignment is not None:
is_expired = False
except KeyError: # planning is not enabled
pass
if is_expired:
# now check recursively for all references
if item.get(config.ID_FIELD) in processed_item:
return is_expired
processed_item[item.get(config.ID_FIELD)] = item
if item_refs:
archive_items = archive_service.get_from_mongo(
req=None, lookup={'_id': {
'$in': item_refs
}})
for archive_item in archive_items:
is_expired = self._can_remove_item(
archive_item, processed_item, preserve_published_desks)
if not is_expired:
break
# If this item is not expired then it is potentially keeping it's parent alive.
if not is_expired:
logger.info('{} Item ID: [{}] has not expired'.format(
self.log_msg, item.get(config.ID_FIELD)))
return is_expired
def _can_remove_item(self,
item,
processed_item=None,
preserve_published_desks=None):
"""Recursively checks if the item can be removed.
:param dict item: item to be remove
:param set processed_item: processed items
:return: True if item can be removed, False otherwise.
"""
if processed_item is None:
processed_item = dict()
item_refs = []
package_service = PackageService()
archive_service = get_resource_service(ARCHIVE)
if item.get(ITEM_TYPE) == CONTENT_TYPE.COMPOSITE:
# Get the item references for is package
item_refs = package_service.get_residrefs(item)
if item.get(ITEM_TYPE) in [
CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED
]:
broadcast_items = get_resource_service(
'archive_broadcast').get_broadcast_items_from_master_story(
item)
# If master story expires then check if broadcast item is included in a package.
# If included in a package then check the package expiry.
item_refs.extend([
broadcast_item.get(config.ID_FIELD)
for broadcast_item in broadcast_items
])
if item.get('rewrite_of'):
item_refs.append(item.get('rewrite_of'))
if item.get('rewritten_by'):
item_refs.append(item.get('rewritten_by'))
# get the list of associated item ids
if item.get(ITEM_TYPE) in MEDIA_TYPES:
item_refs.extend(self._get_associated_items(item))
# get item reference where this referred
item_refs.extend(package_service.get_linked_in_package_ids(item))
# check item refs in the ids to remove set
is_expired = item.get('expiry') and item.get('expiry') < utcnow()
# if the item is published or corrected and desk has preserve_published_content as true
if preserve_published_desks and \
item.get(ITEM_STATE) in {CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED} and \
item.get('task').get('desk') in preserve_published_desks:
is_expired = False
if is_expired:
# now check recursively for all references
if item.get(config.ID_FIELD) in processed_item:
return is_expired
processed_item[item.get(config.ID_FIELD)] = item
if item_refs:
archive_items = archive_service.get_from_mongo(
req=None, lookup={'_id': {
'$in': item_refs
}})
for archive_item in archive_items:
is_expired = self._can_remove_item(
archive_item, processed_item, preserve_published_desks)
if not is_expired:
break
return is_expired
def _removed_refs_from_package(self, item):
"""Remove reference from the package of the spiked item.
:param item:
"""
PackageService().remove_spiked_refs_from_package(item)
from apps.content import push_content_notification
from apps.auth import get_user_id
from superdesk import get_resource_service
from superdesk.errors import SuperdeskApiError, InvalidStateTransitionError
from superdesk.metadata.item import CONTENT_STATE, ITEM_STATE
from superdesk.metadata.packages import RESIDREF
from superdesk.resource import Resource
from superdesk.services import BaseService
from superdesk.workflow import is_workflow_state_transition_valid
from superdesk.utc import utcnow
from apps.packages import PackageService
from flask_babel import _
from flask import current_app as app
package_service = PackageService()
class TranslateResource(Resource):
endpoint_name = "translate"
resource_title = endpoint_name
schema = {
"guid": {"type": "string", "required": True},
"language": {"type": "string", "required": True},
"desk": Resource.rel("desks", nullable=True),
}
url = "archive/translate"
resource_methods = ["POST"]
class ArchiveBroadcastService(BaseService):
packageService = PackageService()
def create(self, docs):
service = get_resource_service(SOURCE)
item_id = request.view_args['item_id']
item = service.find_one(req=None, _id=item_id)
doc = docs[0]
self._valid_broadcast_item(item)
desk_id = doc.get('desk')
desk = None
if desk_id:
desk = get_resource_service('desks').find_one(req=None,
_id=desk_id)
doc.pop('desk', None)
doc['task'] = {}
if desk:
doc['task']['desk'] = desk.get(config.ID_FIELD)
doc['task']['stage'] = desk.get('working_stage')
doc['task']['user'] = get_user().get('_id')
genre_list = get_resource_service('vocabularies').find_one(
req=None, _id='genre') or {}
broadcast_genre = [
{
'qcode': genre.get('qcode'),
'name': genre.get('name')
} for genre in genre_list.get('items', [])
if genre.get('qcode') == BROADCAST_GENRE and genre.get('is_active')
]
if not broadcast_genre:
raise SuperdeskApiError.badRequestError(
message=_("Cannot find the {genre} genre.").format(
genre=BROADCAST_GENRE))
doc['broadcast'] = {
'status': '',
'master_id': item_id,
'rewrite_id': item.get('rewritten_by')
}
doc['genre'] = broadcast_genre
doc['family_id'] = item.get('family_id')
for key in FIELDS_TO_COPY:
doc[key] = item.get(key)
resolve_document_version(document=doc, resource=SOURCE, method='POST')
service.post(docs)
insert_into_versions(id_=doc[config.ID_FIELD])
build_custom_hateoas(CUSTOM_HATEOAS, doc)
return [doc[config.ID_FIELD]]
def _valid_broadcast_item(self, item):
"""Validates item for broadcast.
Broadcast item can only be created for Text or Pre-formatted item.
Item state needs to be Published or Corrected
:param dict item: Item from which the broadcast item will be created
"""
if not item:
raise SuperdeskApiError.notFoundError(
message=_("Cannot find the requested item id."))
if not item.get(ITEM_TYPE) in [
CONTENT_TYPE.TEXT, CONTENT_TYPE.PREFORMATTED
]:
raise SuperdeskApiError.badRequestError(
message=_("Invalid content type."))
if item.get(ITEM_STATE) not in [
CONTENT_STATE.CORRECTED, CONTENT_STATE.PUBLISHED
]:
raise SuperdeskApiError.badRequestError(
message=_("Invalid content state."))
def _get_broadcast_items(self, ids, include_archived_repo=False):
"""Returns list of broadcast items.
Get the broadcast items for the master_id
:param list ids: list of item ids
:param include_archived_repo True if archived repo needs to be included in search, default is False
:return list: list of broadcast items
"""
query = {
'query': {
'bool': {
'filter': [
{
'term': {
'genre.name': BROADCAST_GENRE
}
},
{
'terms': {
'broadcast.master_id': ids
}
},
],
}
}
}
req = ParsedRequest()
repos = 'archive,published'
if include_archived_repo:
repos = 'archive,published,archived'
req.args = {'source': json.dumps(query), 'repo': repos}
return get_resource_service('search').get(req=req, lookup=None)
def get_broadcast_items_from_master_story(self,
item,
include_archived_repo=False):
"""Get the broadcast items from the master story.
:param dict item: master story item
:param include_archived_repo True if archived repo needs to be included in search, default is False
:return list: returns list of broadcast items
"""
if is_genre(item, BROADCAST_GENRE):
return []
ids = [str(item.get(config.ID_FIELD))]
return list(self._get_broadcast_items(ids, include_archived_repo))
def on_broadcast_master_updated(self, item_event, item, rewrite_id=None):
"""Runs when master item is updated.
This event is called when the master story is corrected, published, re-written
:param str item_event: Item operations
:param dict item: item on which operation performed.
:param str rewrite_id: re-written story id.
"""
status = ''
if not item or is_genre(item, BROADCAST_GENRE):
return
elif item_event == ITEM_CREATE and rewrite_id:
status = 'Master Story Re-written'
elif item_event == ITEM_PUBLISH:
status = 'Master Story Published'
elif item_event == ITEM_CORRECT:
status = 'Master Story Corrected'
broadcast_items = self.get_broadcast_items_from_master_story(item)
if not broadcast_items:
return
processed_ids = set()
for broadcast_item in broadcast_items:
try:
if broadcast_item.get('lock_user'):
continue
updates = {
'broadcast': broadcast_item.get('broadcast'),
}
if status:
updates['broadcast']['status'] = status
if not updates['broadcast']['rewrite_id'] and rewrite_id:
updates['broadcast']['rewrite_id'] = rewrite_id
if not broadcast_item.get(config.ID_FIELD) in processed_ids:
self._update_broadcast_status(broadcast_item, updates)
# list of ids that are processed.
processed_ids.add(broadcast_item.get(config.ID_FIELD))
except Exception:
logger.exception(
'Failed to update status for the broadcast item {}'.format(
broadcast_item.get(config.ID_FIELD)))
def _update_broadcast_status(self, item, updates):
"""Update the status of the broadcast item.
:param dict item: broadcast item to be updated
:param dict updates: broadcast updates
"""
# update the published collection as well as archive.
if item.get(ITEM_STATE) in \
{CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED, CONTENT_STATE.KILLED, CONTENT_STATE.RECALLED}:
get_resource_service('published').update_published_items(
item.get(config.ID_FIELD), 'broadcast',
updates.get('broadcast'))
archive_item = get_resource_service(SOURCE).find_one(
req=None, _id=item.get(config.ID_FIELD))
get_resource_service(SOURCE).system_update(
archive_item.get(config.ID_FIELD), updates, archive_item)
def remove_rewrite_refs(self, item):
"""Remove the rewrite references from the broadcast item if the re-write is spiked.
:param dict item: Re-written article of the original story
"""
if is_genre(item, BROADCAST_GENRE):
return
query = {
'query': {
'bool': {
'filter': [
{
'term': {
'genre.name': BROADCAST_GENRE
}
},
{
'term': {
'broadcast.rewrite_id':
item.get(config.ID_FIELD)
}
},
]
}
}
}
req = ParsedRequest()
req.args = {'source': json.dumps(query)}
broadcast_items = list(
get_resource_service(SOURCE).get(req=req, lookup=None))
for broadcast_item in broadcast_items:
try:
updates = {'broadcast': broadcast_item.get('broadcast', {})}
updates['broadcast']['rewrite_id'] = None
if 'Re-written' in updates['broadcast']['status']:
updates['broadcast']['status'] = ''
self._update_broadcast_status(broadcast_item, updates)
except Exception:
logger.exception(
'Failed to remove rewrite id for the broadcast item {}'.
format(broadcast_item.get(config.ID_FIELD)))
def reset_broadcast_status(self, updates, original):
"""Reset the broadcast status if the broadcast item is updated.
:param dict updates: updates to the original document
:param dict original: original document
"""
if original.get('broadcast') and original.get('broadcast').get(
'status', ''):
broadcast_updates = {
'broadcast': original.get('broadcast'),
}
broadcast_updates['broadcast']['status'] = ''
self._update_broadcast_status(original, broadcast_updates)
updates.update(broadcast_updates)
def spike_item(self, original):
"""If Original item is re-write then it will remove the reference from the broadcast item.
:param: dict original: original document
"""
broadcast_items = [
item
for item in self.get_broadcast_items_from_master_story(original)
if item.get(ITEM_STATE) not in PUBLISH_STATES
]
spike_service = get_resource_service('archive_spike')
for item in broadcast_items:
id_ = item.get(config.ID_FIELD)
try:
self.packageService.remove_spiked_refs_from_package(id_)
updates = {ITEM_STATE: CONTENT_STATE.SPIKED}
resolve_document_version(updates, SOURCE, 'PATCH', item)
spike_service.patch(id_, updates)
insert_into_versions(id_=id_)
except Exception:
logger.exception(
message="Failed to spike the related broadcast item {}.".
format(id_))
if original.get('rewrite_of') and original.get(
ITEM_STATE) not in PUBLISH_STATES:
self.remove_rewrite_refs(original)
def kill_broadcast(self, updates, original, operation):
"""Kill the broadcast items
:param dict updates: Updates to the item
:param dict original: original item
:param str operation: Kill or Takedown operation
:return:
"""
broadcast_items = [
item
for item in self.get_broadcast_items_from_master_story(original)
if item.get(ITEM_STATE) in PUBLISH_STATES
]
correct_service = get_resource_service('archive_correct')
kill_service = get_resource_service('archive_{}'.format(operation))
for item in broadcast_items:
item_id = item.get(config.ID_FIELD)
packages = self.packageService.get_packages(item_id)
processed_packages = set()
for package in packages:
if str(package[config.ID_FIELD]) in processed_packages or \
package.get(ITEM_STATE) == CONTENT_STATE.RECALLED:
continue
try:
if package.get(ITEM_STATE) in {
CONTENT_STATE.PUBLISHED, CONTENT_STATE.CORRECTED
}:
package_updates = {
config.LAST_UPDATED:
utcnow(),
GROUPS:
self.packageService.remove_group_ref(
package, item_id)
}
refs = self.packageService.get_residrefs(
package_updates)
if refs:
correct_service.patch(package.get(config.ID_FIELD),
package_updates)
else:
package_updates['body_html'] = updates.get(
'body_html', '')
kill_service.patch(package.get(config.ID_FIELD),
package_updates)
processed_packages.add(package.get(config.ID_FIELD))
else:
package_list = self.packageService.remove_refs_in_package(
package, item_id, processed_packages)
processed_packages = processed_packages.union(
set(package_list))
except Exception:
logger.exception(
'Failed to remove the broadcast item {} from package {}'
.format(item_id, package.get(config.ID_FIELD)))
kill_service.kill_item(updates, item)