def create(self, docs, **kwargs):
new_guids = []
provider = get_resource_service('ingest_providers').find_one(
source='aapmm', req=None)
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError(
"Destination desk cannot be empty.")
archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(
provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(dest_doc, doc.get('desk'), doc.get('stage'))
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
return new_guids
def create(self, docs, **kwargs):
new_guids = []
provider = self.get_provider()
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
try:
archived_doc = self.fetch(doc['guid'])
except FileNotFoundError as ex:
raise ProviderError.externalProviderError(ex, provider)
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
return new_guids
def create(self, docs, **kwargs):
new_guids = []
provider = get_resource_service("ingest_providers").find_one(source="aapmm", req=None)
if provider and "config" in provider and "username" in provider["config"]:
self.backend.set_credentials(provider["config"]["username"], provider["config"]["password"])
for doc in docs:
if not doc.get("desk"):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
try:
archived_doc = self.backend.find_one_raw(doc["guid"], doc["guid"])
except FileNotFoundError as ex:
raise ProviderError.externalProviderError(ex, provider)
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc["_id"] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc["ingest_provider"] = str(provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc, update=None, desk_id=doc.get("desk"), stage_id=doc.get("stage"))
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = archived_doc["_id"]
dest_doc[FAMILY_ID] = archived_doc["_id"]
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get("_id"))
return new_guids
def create(self, docs, **kwargs):
new_guids = []
provider = get_resource_service('ingest_providers').find_one(source='aapmm', req=None)
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
return new_guids
def fetch(self, docs, id=None, **kwargs):
id_of_fetched_items = []
for doc in docs:
id_of_item_to_be_fetched = doc.get('_id') if id is None else id
desk_id = doc.get('desk')
stage_id = doc.get('stage')
ingest_service = get_resource_service('ingest')
ingest_doc = ingest_service.find_one(req=None,
_id=id_of_item_to_be_fetched)
if not ingest_doc:
raise SuperdeskApiError.notFoundError(
'Fail to found ingest item with _id: %s' %
id_of_item_to_be_fetched)
if not is_workflow_state_transition_valid(
'fetch_from_ingest', ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
if doc.get('macro'): # there is a macro so transform it
ingest_doc = get_resource_service('macros').execute_macro(
ingest_doc, doc.get('macro'))
archived = utcnow()
ingest_service.patch(id_of_item_to_be_fetched,
{'archived': archived})
dest_doc = dict(ingest_doc)
new_id = generate_guid(type=GUID_TAG)
id_of_fetched_items.append(new_id)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
dest_doc['destination_groups'] = doc.get('destination_groups')
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, desk_id, stage_id)
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
self.__fetch_items_in_package(dest_doc, desk_id, stage_id,
doc.get('state', STATE_FETCHED),
doc.get('destination_groups'))
get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(doc=dest_doc)
build_custom_hateoas(custom_hateoas, dest_doc)
doc.update(dest_doc)
if kwargs.get('notify', True):
push_notification('item:fetch', fetched=1)
return id_of_fetched_items
def _publish_associated_items(self, original, updates={}, publish=False):
"""Refresh and publish associated items before publishing. The publishing is done if the setting
PUBLISH_ASSOCIATED_ITEMS was true.
Any further updates made to basic metadata done after item was associated will be carried on and
used when validating those items.
"""
publish_service = None
if config.PUBLISH_ASSOCIATED_ITEMS and publish_services.get(
self.publish_type):
publish_service = get_resource_service(
publish_services[self.publish_type])
associations = original.get(ASSOCIATIONS) or {}
for associations_key, item in associations.items():
if type(item) == dict and item.get(config.ID_FIELD):
keys = [
key for key in DEFAULT_SCHEMA.keys()
if key not in PRESERVED_FIELDS
]
if app.settings.get('COPY_METADATA_FROM_PARENT') and item.get(
ITEM_TYPE) in MEDIA_TYPES:
original_item = original
keys = FIELDS_TO_COPY_FOR_ASSOCIATED_ITEM
else:
original_item = super().find_one(
req=None, _id=item[config.ID_FIELD]) or {}
update_item_data(item, original_item, keys)
remove_unwanted(item)
if publish_service and publish and item['type'] in MEDIA_TYPES:
if item.get('task', {}).get('stage', None):
del item['task']['stage']
if item['state'] not in PUBLISH_STATES:
get_resource_service('archive_publish').patch(
id=item.pop(config.ID_FIELD), updates=item)
else:
publish_service.patch(id=item.pop(config.ID_FIELD),
updates=item)
item['state'] = self.published_state
original[ASSOCIATIONS][associations_key][
'state'] = self.published_state
original[ASSOCIATIONS][associations_key][
'operation'] = self.publish_type
if ASSOCIATIONS not in updates:
updates[ASSOCIATIONS] = original[ASSOCIATIONS]
updates[ASSOCIATIONS][associations_key] = original[
ASSOCIATIONS][associations_key]
elif publish:
# Publishing an item with an association set the state and operation of the embedded association
original[ASSOCIATIONS][associations_key][
'state'] = self.published_state
original[ASSOCIATIONS][associations_key][
'operation'] = self.publish_type
def create(self, docs, **kwargs):
search_provider = get_resource_service('search_providers').find_one(
search_provider=PROVIDER_NAME, req=None)
if not search_provider or search_provider.get('is_closed', False):
raise SuperdeskApiError.badRequestError(
'No search provider found or the search provider is closed.')
if 'config' in search_provider:
self.backend.set_credentials(search_provider['config'])
new_guids = []
for doc in docs:
if not doc.get(
'desk'): # if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError(
"Destination desk cannot be empty.")
try:
archived_doc = self.backend.find_one_raw(
doc['guid'], doc['guid'])
except FileNotFoundError as ex:
raise ProviderError.externalProviderError(ex, search_provider)
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc[config.ID_FIELD] = new_id
generate_unique_id_and_name(dest_doc)
if search_provider:
dest_doc['ingest_provider'] = str(
search_provider[config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc,
update=None,
desk_id=doc.get('desk'),
stage_id=doc.get('stage'))
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = archived_doc[config.ID_FIELD]
dest_doc[FAMILY_ID] = archived_doc[config.ID_FIELD]
dest_doc[ITEM_OPERATION] = ITEM_FETCH
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc[config.ID_FIELD])
get_resource_service('search_providers').system_update(
search_provider[config.ID_FIELD],
{'last_item_update': utcnow()}, search_provider)
return new_guids
def fetch(self, docs, id=None, **kwargs):
id_of_fetched_items = []
for doc in docs:
id_of_item_to_be_fetched = doc.get('_id') if id is None else id
desk_id = doc.get('desk')
stage_id = doc.get('stage')
ingest_service = get_resource_service('ingest')
ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched)
if not ingest_doc:
raise SuperdeskApiError.notFoundError('Fail to found ingest item with _id: %s' %
id_of_item_to_be_fetched)
if not is_workflow_state_transition_valid('fetch_from_ingest', ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
if doc.get('macro'): # there is a macro so transform it
ingest_doc = get_resource_service('macros').execute_macro(ingest_doc, doc.get('macro'))
archived = utcnow()
ingest_service.patch(id_of_item_to_be_fetched, {'archived': archived})
dest_doc = dict(ingest_doc)
new_id = generate_guid(type=GUID_TAG)
id_of_fetched_items.append(new_id)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
dest_doc['destination_groups'] = doc.get('destination_groups')
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, desk_id, stage_id)
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
self.__fetch_items_in_package(dest_doc, desk_id, stage_id,
doc.get('state', STATE_FETCHED),
doc.get('destination_groups'))
get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(doc=dest_doc)
build_custom_hateoas(custom_hateoas, dest_doc)
doc.update(dest_doc)
if kwargs.get('notify', True):
push_notification('item:fetch', fetched=1)
return id_of_fetched_items
def fetch(self, docs, id=None, **kwargs):
id_of_fetched_items = []
for doc in docs:
id_of_item_to_be_fetched = doc.get("_id") if id is None else id
desk_id = doc.get("desk")
stage_id = doc.get("stage")
ingest_service = get_resource_service("ingest")
ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched)
if not ingest_doc:
raise SuperdeskApiError.notFoundError(
"Fail to found ingest item with _id: %s" % id_of_item_to_be_fetched
)
if not is_workflow_state_transition_valid("fetch_from_ingest", ingest_doc[ITEM_STATE]):
raise InvalidStateTransitionError()
if doc.get("macro"): # there is a macro so transform it
ingest_doc = get_resource_service("macros").execute_macro(ingest_doc, doc.get("macro"))
archived = utcnow()
ingest_service.patch(id_of_item_to_be_fetched, {"archived": archived})
dest_doc = dict(ingest_doc)
new_id = generate_guid(type=GUID_TAG)
id_of_fetched_items.append(new_id)
dest_doc["_id"] = new_id
dest_doc["guid"] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc, desk_id=desk_id, stage_id=stage_id)
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc["_id"]
dest_doc[ITEM_OPERATION] = ITEM_FETCH
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
self.__fetch_items_in_package(dest_doc, desk_id, stage_id, doc.get(ITEM_STATE, CONTENT_STATE.FETCHED))
get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(doc=dest_doc)
build_custom_hateoas(custom_hateoas, dest_doc)
doc.update(dest_doc)
if kwargs.get("notify", True):
push_notification("item:fetch", fetched=1)
return id_of_fetched_items
def _update_archive(guid, item):
"""
Assigns a State to the content, removes unwanted attributes, sets the original creator and updates the item in
archive collection.
:param guid:
:param item: from ingest collection
"""
item[config.CONTENT_STATE] = STATE_FETCHED
remove_unwanted(item)
set_original_creator(item)
superdesk.get_resource_service(ARCHIVE).update(guid, item)
def create(self, docs, **kwargs):
for doc in docs:
ingest_doc = superdesk.get_resource_service("ingest").find_one(req=None, _id=doc.get("guid"))
if not ingest_doc:
msg = "Fail to found ingest item with guid: %s" % doc.get("guid")
raise SuperdeskApiError.notFoundError(msg)
if not is_workflow_state_transition_valid("fetch_as_from_ingest", ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
archived = utcnow()
superdesk.get_resource_service("ingest").patch(ingest_doc.get("_id"), {"archived": archived})
doc["archived"] = archived
archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, _id=doc.get("guid"))
if not archived_doc:
dest_doc = dict(ingest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, doc.get("desk"))
dest_doc[config.CONTENT_STATE] = STATE_FETCHED
remove_unwanted(dest_doc)
for ref in [
ref for group in dest_doc.get("groups", []) for ref in group.get("refs", []) if "residRef" in ref
]:
ref["location"] = ARCHIVE
ref["guid"] = ref["residRef"]
set_original_creator(dest_doc)
if doc.get(PACKAGE):
links = dest_doc.get(LINKED_IN_PACKAGES, [])
links.append({PACKAGE: doc.get(PACKAGE)})
dest_doc[LINKED_IN_PACKAGES] = links
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get("guid"))
desk = doc.get("desk")
refs = [
{"guid": ref.get("residRef"), "desk": desk, PACKAGE: dest_doc.get("_id")}
for group in dest_doc.get("groups", [])
for ref in group.get("refs", [])
if "residRef" in ref
]
if refs:
self.create(refs)
else:
if doc.get(PACKAGE):
links = archived_doc.get(LINKED_IN_PACKAGES, [])
links.append({PACKAGE: doc.get(PACKAGE)})
superdesk.get_resource_service(ARCHIVE).patch(archived_doc.get("_id"), {LINKED_IN_PACKAGES: links})
return [doc.get("guid") for doc in docs]
def ingest_items_for(self, desk, no_of_stories, skip_index):
desk_id = desk['_id']
stage_id = desk['incoming_stage']
bucket_size = min(100, no_of_stories)
no_of_buckets = len(range(0, no_of_stories, bucket_size))
for x in range(0, no_of_buckets):
skip = x * bucket_size * skip_index
logger.info('Page : {}, skip: {}'.format(x + 1, skip))
cursor = get_resource_service('published').get_from_mongo(None, {})
cursor.skip(skip)
cursor.limit(bucket_size)
items = list(cursor)
logger.info('Inserting {} items'.format(len(items)))
archive_items = []
for item in items:
dest_doc = dict(item)
new_id = generate_guid(type=GUID_TAG)
dest_doc[app.config['ID_FIELD']] = new_id
dest_doc['guid'] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[app.config['VERSION']] = 1
dest_doc[ITEM_STATE] = CONTENT_STATE.FETCHED
user_id = desk.get('members', [{'user': None}])[0].get('user')
dest_doc['original_creator'] = user_id
dest_doc['version_creator'] = user_id
from apps.tasks import send_to
send_to(dest_doc,
desk_id=desk_id,
stage_id=stage_id,
user_id=user_id)
dest_doc[app.config[
'VERSION']] = 1 # Above step increments the version and needs to reset
dest_doc[FAMILY_ID] = item['_id']
remove_unwanted(dest_doc)
archive_items.append(dest_doc)
get_resource_service(ARCHIVE).post(archive_items)
for item in archive_items:
insert_into_versions(id_=item[app.config['ID_FIELD']])
def create(self, docs, **kwargs):
for doc in docs:
ingest_doc = superdesk.get_resource_service('ingest').find_one(req=None, _id=doc.get('guid'))
if not ingest_doc:
msg = 'Fail to found ingest item with guid: %s' % doc.get('guid')
raise SuperdeskApiError.notFoundError(msg)
if not is_workflow_state_transition_valid('fetch_as_from_ingest', ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
archived = utcnow()
superdesk.get_resource_service('ingest').patch(ingest_doc.get('_id'), {'archived': archived})
doc['archived'] = archived
archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, _id=doc.get('guid'))
if not archived_doc:
dest_doc = dict(ingest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, doc.get('desk'))
dest_doc[config.CONTENT_STATE] = STATE_FETCHED
remove_unwanted(dest_doc)
for ref in [ref for group in dest_doc.get('groups', [])
for ref in group.get('refs', []) if 'residRef' in ref]:
ref['location'] = ARCHIVE
ref['guid'] = ref['residRef']
set_original_creator(dest_doc)
if doc.get(PACKAGE):
links = dest_doc.get(LINKED_IN_PACKAGES, [])
links.append({PACKAGE: doc.get(PACKAGE)})
dest_doc[LINKED_IN_PACKAGES] = links
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('guid'))
desk = doc.get('desk')
refs = [{'guid': ref.get('residRef'), 'desk': desk, PACKAGE: dest_doc.get('_id')}
for group in dest_doc.get('groups', [])
for ref in group.get('refs', []) if 'residRef' in ref]
if refs:
self.create(refs)
else:
if doc.get(PACKAGE):
links = archived_doc.get(LINKED_IN_PACKAGES, [])
links.append({PACKAGE: doc.get(PACKAGE)})
superdesk.get_resource_service(ARCHIVE).patch(archived_doc.get('_id'), {LINKED_IN_PACKAGES: links})
return [doc.get('guid') for doc in docs]
def create(self, docs, **kwargs):
new_guids = []
provider = get_resource_service('ingest_providers').find_one(
source='aapmm', req=None)
if provider and 'config' in provider and 'username' in provider[
'config']:
self.backend.set_credentials(provider['config']['username'],
provider['config']['password'])
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError(
"Destination desk cannot be empty.")
try:
archived_doc = self.backend.find_one_raw(
doc['guid'], doc['guid'])
except FileNotFoundError as ex:
raise ProviderError.externalProviderError(ex, provider)
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(
provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc,
update=None,
desk_id=doc.get('desk'),
stage_id=doc.get('stage'))
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
return new_guids
def create(self, docs, **kwargs):
search_provider = get_resource_service('search_providers').find_one(search_provider=PROVIDER_NAME, req=None)
if not search_provider or search_provider.get('is_closed', False):
raise SuperdeskApiError.badRequestError('No search provider found or the search provider is closed.')
if 'config' in search_provider:
self.backend.set_credentials(search_provider['config'])
new_guids = []
for doc in docs:
if not doc.get('desk'): # if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
try:
archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])
except FileNotFoundError as ex:
raise ProviderError.externalProviderError(ex, search_provider)
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc[config.ID_FIELD] = new_id
generate_unique_id_and_name(dest_doc)
if search_provider:
dest_doc['ingest_provider'] = str(search_provider[config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = archived_doc[config.ID_FIELD]
dest_doc[FAMILY_ID] = archived_doc[config.ID_FIELD]
dest_doc[ITEM_OPERATION] = ITEM_FETCH
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc[config.ID_FIELD])
get_resource_service('search_providers').system_update(search_provider[config.ID_FIELD],
{'last_item_update': utcnow()}, search_provider)
return new_guids
def create(self, docs, **kwargs):
new_guids = []
provider = self.get_provider()
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError(
_("Destination desk cannot be empty."))
try:
archived_doc = self.fetch(doc['guid'])
except FileNotFoundError as ex:
raise ProviderError.externalProviderError(ex, provider)
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(
provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc,
update=None,
desk_id=doc.get('desk'),
stage_id=doc.get('stage'))
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
dest_doc[ITEM_OPERATION] = ITEM_FETCH
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
if new_guids:
get_resource_service('search_providers').system_update(
provider.get(config.ID_FIELD), {'last_item_update': utcnow()},
provider)
return new_guids
def ingest_items_for(self, desk, no_of_stories, skip_index):
desk_id = desk['_id']
stage_id = desk['incoming_stage']
bucket_size = min(100, no_of_stories)
no_of_buckets = len(range(0, no_of_stories, bucket_size))
for x in range(0, no_of_buckets):
skip = x * bucket_size * skip_index
logger.info('Page : {}, skip: {}'.format(x + 1, skip))
cursor = get_resource_service('published').get_from_mongo(None, {})
cursor.skip(skip)
cursor.limit(bucket_size)
items = list(cursor)
logger.info('Inserting {} items'.format(len(items)))
archive_items = []
for item in items:
dest_doc = dict(item)
new_id = generate_guid(type=GUID_TAG)
dest_doc[app.config['ID_FIELD']] = new_id
dest_doc['guid'] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[app.config['VERSION']] = 1
dest_doc[ITEM_STATE] = CONTENT_STATE.FETCHED
user_id = desk.get('members', [{'user': None}])[0].get('user')
dest_doc['original_creator'] = user_id
dest_doc['version_creator'] = user_id
from apps.tasks import send_to
send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id)
dest_doc[app.config['VERSION']] = 1 # Above step increments the version and needs to reset
dest_doc[FAMILY_ID] = item['_id']
remove_unwanted(dest_doc)
archive_items.append(dest_doc)
get_resource_service(ARCHIVE).post(archive_items)
for item in archive_items:
insert_into_versions(id_=item[app.config['ID_FIELD']])
def _publish_associated_items(self, original, updates={}, publish=False):
"""Refresh and publish associated items before publishing. The publishing is done if the setting
PUBLISH_ASSOCIATED_ITEMS was true.
Any further updates made to basic metadata done after item was associated will be carried on and
used when validating those items.
"""
publish_service = None
if config.PUBLISH_ASSOCIATED_ITEMS and publish_services.get(self.publish_type):
publish_service = get_resource_service(publish_services[self.publish_type])
associations = original.get(ASSOCIATIONS) or {}
for associations_key, item in associations.items():
if type(item) == dict and item.get(config.ID_FIELD):
keys = [key for key in DEFAULT_SCHEMA.keys() if key not in PRESERVED_FIELDS]
if app.settings.get('COPY_METADATA_FROM_PARENT') and item.get(ITEM_TYPE) in MEDIA_TYPES:
original_item = original
keys = FIELDS_TO_COPY_FOR_ASSOCIATED_ITEM
else:
original_item = super().find_one(req=None, _id=item[config.ID_FIELD]) or {}
update_item_data(item, original_item, keys)
remove_unwanted(item)
if publish_service and publish and item['type'] in MEDIA_TYPES:
if item.get('task', {}).get('stage', None):
del item['task']['stage']
if item['state'] not in PUBLISH_STATES:
get_resource_service('archive_publish').patch(id=item.pop(config.ID_FIELD), updates=item)
else:
publish_service.patch(id=item.pop(config.ID_FIELD), updates=item)
item['state'] = self.published_state
original[ASSOCIATIONS][associations_key]['state'] = self.published_state
original[ASSOCIATIONS][associations_key]['operation'] = self.publish_type
if ASSOCIATIONS not in updates:
updates[ASSOCIATIONS] = original[ASSOCIATIONS]
updates[ASSOCIATIONS][associations_key] = original[ASSOCIATIONS][associations_key]
elif publish:
# Publishing an item with an association set the state and operation of the embedded association
original[ASSOCIATIONS][associations_key]['state'] = self.published_state
original[ASSOCIATIONS][associations_key]['operation'] = self.publish_type
def ingest_items_for(self, desk, no_of_stories, skip_index):
desk_id = desk['_id']
stage_id = desk['incoming_stage']
bucket_size = min(100, no_of_stories)
no_of_buckets = len(range(0, no_of_stories, bucket_size))
for x in range(0, no_of_buckets):
skip = x * bucket_size * skip_index
self.logger.info('Page : {}, skip: {}'.format(x + 1, skip))
cursor = get_resource_service('text_archive').get_from_mongo(None, {})
cursor.skip(skip)
cursor.limit(bucket_size)
items = list(cursor)
self.logger.info('Inserting {} items'.format(len(items)))
archive_items = []
for item in items:
dest_doc = dict(item)
new_id = generate_guid(type=GUID_TAG)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[app.config['VERSION']] = 1
dest_doc['state'] = 'fetched'
user_id = desk.get('members', [{'user': None}])[0].get('user')
dest_doc['original_creator'] = user_id
dest_doc['version_creator'] = user_id
send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id)
dest_doc[FAMILY_ID] = item['_id']
remove_unwanted(dest_doc)
archive_items.append(dest_doc)
get_resource_service(ARCHIVE).post(archive_items)
for item in archive_items:
insert_into_versions(id_=item['_id'])
def _publish_associated_items(self, original, updates=None):
"""If there any updates to associated item and if setting:PUBLISH_ASSOCIATED_ITEMS is true
then publish the associated item
"""
if updates is None:
updates = {}
if not publish_services.get(self.publish_type):
# publish type not supported
return
publish_service = get_resource_service(
publish_services.get(self.publish_type))
if not updates.get(ASSOCIATIONS) and not original.get(ASSOCIATIONS):
# there's nothing to update
return
associations = original.get(ASSOCIATIONS) or {}
if updates and updates.get(ASSOCIATIONS):
associations.update(updates[ASSOCIATIONS])
for associations_key, associated_item in associations.items():
if associated_item is None:
continue
if type(associated_item) == dict and associated_item.get(
config.ID_FIELD):
if not config.PUBLISH_ASSOCIATED_ITEMS or not publish_service:
# Not allowed to publish
original[ASSOCIATIONS][associations_key][
'state'] = self.published_state
original[ASSOCIATIONS][associations_key][
'operation'] = self.publish_type
continue
# if item is not fetchable, only mark it as published
if not associated_item.get('_fetchable', True):
associated_item['state'] = self.published_state
associated_item['operation'] = self.publish_type
updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
updates[ASSOCIATIONS][associations_key] = associated_item
continue
if associated_item.get('state') == CONTENT_STATE.UNPUBLISHED:
# get the original associated item from archive
orig_associated_item = get_resource_service('archive') \
.find_one(req=None, _id=associated_item[config.ID_FIELD])
orig_associated_item['state'] = self.published_state
orig_associated_item['operation'] = self.publish_type
get_resource_service('archive_publish').patch(
id=orig_associated_item.pop(config.ID_FIELD),
updates=orig_associated_item)
continue
if associated_item.get('state') not in PUBLISH_STATES:
# This associated item has not been published before
associated_item.get('task', {}).pop('stage', None)
remove_unwanted(associated_item)
# get the original associated item from archive
original_associated_item = get_resource_service('archive').\
find_one(req=None, _id=associated_item[config.ID_FIELD])
# check if the original associated item exists in archive
if not original_associated_item:
raise SuperdeskApiError.badRequestError(
_('Associated item "{}" does not exist in the system'
.format(associations_key)))
if original_associated_item.get('state') in PUBLISH_STATES:
# item was published already
original[ASSOCIATIONS][associations_key].update({
'state':
original_associated_item['state'],
'operation':
original_associated_item.get(
'operation', self.publish_type),
})
continue
# update _updated, otherwise it's stored as string.
# fixes SDESK-5043
associated_item['_updated'] = utcnow()
get_resource_service('archive_publish').patch(
id=associated_item.pop(config.ID_FIELD),
updates=associated_item)
associated_item['state'] = self.published_state
associated_item['operation'] = self.publish_type
updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
updates[ASSOCIATIONS][associations_key] = associated_item
elif associated_item.get('state') != self.published_state:
# Check if there are updates to associated item
association_updates = updates.get(ASSOCIATIONS,
{}).get(associations_key)
if not association_updates:
# there is no update for this item
associated_item.get('task', {}).pop('stage', None)
remove_unwanted(associated_item)
publish_service.patch(id=associated_item.pop(
config.ID_FIELD),
updates=associated_item)
continue
if association_updates.get('state') not in PUBLISH_STATES:
# There's an update to the published associated item
remove_unwanted(association_updates)
publish_service.patch(id=association_updates.pop(
config.ID_FIELD),
updates=association_updates)
self._refresh_associated_items(original)
def fetch(self, docs, id=None, **kwargs):
id_of_fetched_items = []
for doc in docs:
id_of_item_to_be_fetched = doc.get(
config.ID_FIELD) if id is None else id
desk_id = doc.get('desk')
stage_id = doc.get('stage')
ingest_service = get_resource_service('ingest')
ingest_doc = ingest_service.find_one(req=None,
_id=id_of_item_to_be_fetched)
if not ingest_doc:
raise SuperdeskApiError.notFoundError(
_('Fail to found ingest item with _id: {id}').format(
id=id_of_item_to_be_fetched))
if not is_workflow_state_transition_valid('fetch_from_ingest',
ingest_doc[ITEM_STATE]):
raise InvalidStateTransitionError()
if doc.get('macro'): # there is a macro so transform it
ingest_doc = get_resource_service('macros').execute_macro(
ingest_doc, doc.get('macro'))
archived = utcnow()
ingest_service.patch(id_of_item_to_be_fetched,
{'archived': archived})
dest_doc = dict(ingest_doc)
if doc.get('target'):
dest_doc.update(doc.get('target'))
new_id = generate_guid(type=GUID_TAG)
id_of_fetched_items.append(new_id)
dest_doc[config.ID_FIELD] = new_id
dest_doc[GUID_FIELD] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
dest_doc['versioncreated'] = archived
send_to(doc=dest_doc, desk_id=desk_id, stage_id=stage_id)
dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc[
config.ID_FIELD]
dest_doc[ITEM_OPERATION] = ITEM_FETCH
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
self.__fetch_items_in_package(
dest_doc, desk_id, stage_id,
doc.get(ITEM_STATE, CONTENT_STATE.FETCHED))
self.__fetch_associated_items(
dest_doc, desk_id, stage_id,
doc.get(ITEM_STATE, CONTENT_STATE.FETCHED))
desk = get_resource_service('desks').find_one(req=None,
_id=desk_id)
if desk and desk.get('default_content_profile'):
dest_doc['profile'] = desk['default_content_profile']
if dest_doc.get('type', 'text') in MEDIA_TYPES:
dest_doc['profile'] = None
get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(doc=dest_doc)
build_custom_hateoas(custom_hateoas, dest_doc)
superdesk.item_fetched.send(self,
item=dest_doc,
ingest_item=ingest_doc)
doc.update(dest_doc)
if kwargs.get('notify', True):
ingest_doc.update({'task': dest_doc.get('task')})
push_item_move_notification(ingest_doc, doc, 'item:fetch')
return id_of_fetched_items
def _publish_associated_items(self, original, updates={}):
"""If there any updates to associated item and if setting:PUBLISH_ASSOCIATED_ITEMS is true
then publish the associated item
"""
if not publish_services.get(self.publish_type):
# publish type not supported
return
publish_service = get_resource_service(
publish_services.get(self.publish_type))
if not updates.get(ASSOCIATIONS) and not original.get(ASSOCIATIONS):
# there's nothing to update
return
associations = original.get(ASSOCIATIONS) or {}
for associations_key, associated_item in associations.items():
if type(associated_item) == dict and associated_item.get(
config.ID_FIELD):
if not config.PUBLISH_ASSOCIATED_ITEMS or not publish_service:
# Not allowed to publish
original[ASSOCIATIONS][associations_key][
'state'] = self.published_state
original[ASSOCIATIONS][associations_key][
'operation'] = self.publish_type
continue
if associated_item.get('state') not in PUBLISH_STATES:
# This associated item has not been published before
associated_item.get('task', {}).pop('stage', None)
remove_unwanted(associated_item)
# get the original associated item from archive
original_associated_item = get_resource_service('archive').\
find_one(req=None, _id=associated_item[config.ID_FIELD])
# check if the original associated item exists in archive
if not original_associated_item:
raise SuperdeskApiError.badRequestError(
_('Associated item "{}" does not exist in the system'
.format(associations_key)))
get_resource_service('archive_publish').patch(
id=associated_item.pop(config.ID_FIELD),
updates=associated_item)
associated_item['state'] = self.published_state
associated_item['operation'] = self.publish_type
updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
updates[ASSOCIATIONS][associations_key] = associated_item
else:
# Check if there are updates to associated item
association_updates = updates.get(ASSOCIATIONS,
{}).get(associations_key)
if not association_updates:
# there is no update for this item but still patching the associated item
associated_item.get('task', {}).pop('stage', None)
remove_unwanted(associated_item)
publish_service.patch(id=associated_item.pop(
config.ID_FIELD),
updates=associated_item)
continue
if association_updates.get('state') not in PUBLISH_STATES:
# There's an update to the published associated item
remove_unwanted(association_updates)
publish_service.patch(id=association_updates.pop(
config.ID_FIELD),
updates=association_updates)
self._refresh_associated_items(original)
def _publish_associated_items(self, original, updates=None):
"""If there any updates to associated item and if setting:PUBLISH_ASSOCIATED_ITEMS is true
then publish the associated item
"""
if updates is None:
updates = {}
if not publish_services.get(self.publish_type):
# publish type not supported
return
publish_service = get_resource_service(publish_services.get(self.publish_type))
if not updates.get(ASSOCIATIONS) and not original.get(ASSOCIATIONS):
# there's nothing to update
return
associations = original.get(ASSOCIATIONS) or {}
if updates and updates.get(ASSOCIATIONS):
associations.update(updates[ASSOCIATIONS])
archive_service = get_resource_service("archive")
for associations_key, associated_item in associations.items():
if associated_item is None:
continue
if type(associated_item) == dict and associated_item.get(config.ID_FIELD):
if not config.PUBLISH_ASSOCIATED_ITEMS or not publish_service:
if original.get(ASSOCIATIONS, {}).get(associations_key):
# Not allowed to publish
original[ASSOCIATIONS][associations_key]["state"] = self.published_state
original[ASSOCIATIONS][associations_key]["operation"] = self.publish_type
continue
# if item is not fetchable, only mark it as published
if not associated_item.get("_fetchable", True):
associated_item["state"] = self.published_state
associated_item["operation"] = self.publish_type
updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
updates[ASSOCIATIONS][associations_key] = associated_item
continue
if associated_item.get("state") == CONTENT_STATE.UNPUBLISHED:
# get the original associated item from archive
orig_associated_item = archive_service.find_one(req=None, _id=associated_item[config.ID_FIELD])
orig_associated_item["state"] = updates.get("state", self.published_state)
orig_associated_item["operation"] = self.publish_type
# if main item is scheduled we must also schedule associations
self._inherit_publish_schedule(original, updates, orig_associated_item)
get_resource_service("archive_publish").patch(
id=orig_associated_item.pop(config.ID_FIELD), updates=orig_associated_item
)
continue
if associated_item.get("state") not in PUBLISH_STATES:
# This associated item has not been published before
remove_unwanted(associated_item)
# get the original associated item from archive
orig_associated_item = archive_service.find_one(req=None, _id=associated_item[config.ID_FIELD])
# check if the original associated item exists in archive
if not orig_associated_item:
raise SuperdeskApiError.badRequestError(
_('Associated item "{}" does not exist in the system'.format(associations_key))
)
if orig_associated_item.get("state") in PUBLISH_STATES:
# item was published already
original[ASSOCIATIONS][associations_key].update(
{
"state": orig_associated_item["state"],
"operation": orig_associated_item.get("operation", self.publish_type),
}
)
continue
# if the original associated item stage is present, it should be updated in the association item.
if orig_associated_item.get("task", {}).get("stage") and associated_item.get("task"):
associated_item["task"].update({"stage": orig_associated_item.get("task", {}).get("stage")})
# update _updated, otherwise it's stored as string.
# fixes SDESK-5043
associated_item["_updated"] = utcnow()
# if main item is scheduled we must also schedule associations
self._inherit_publish_schedule(original, updates, associated_item)
get_resource_service("archive_publish").patch(
id=associated_item.pop(config.ID_FIELD), updates=associated_item
)
associated_item["state"] = updates.get("state", self.published_state)
associated_item["operation"] = self.publish_type
updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
updates[ASSOCIATIONS][associations_key] = associated_item
elif associated_item.get("state") != self.published_state:
# Check if there are updates to associated item
association_updates = updates.get(ASSOCIATIONS, {}).get(associations_key)
# if main item is scheduled we must also schedule associations
self._inherit_publish_schedule(original, updates, associated_item)
if not association_updates:
# there is no update for this item
associated_item.get("task", {}).pop("stage", None)
remove_unwanted(associated_item)
publish_service.patch(id=associated_item.pop(config.ID_FIELD), updates=associated_item)
continue
if association_updates.get("state") not in PUBLISH_STATES:
# There's an update to the published associated item
remove_unwanted(association_updates)
publish_service.patch(id=association_updates.pop(config.ID_FIELD), updates=association_updates)
# When there is an associated item which is published, Inserts the latest version of that associated item into archive_versions.
insert_into_versions(doc=associated_item)
self._refresh_associated_items(original)
def create(self, docs, **kwargs):
new_guids = []
for doc in docs:
ingest_doc = superdesk.get_resource_service('ingest').find_one(req=None, _id=doc.get('guid'))
if not ingest_doc:
# see if it is in archive, if it is duplicate it
archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, _id=doc.get('guid'))
if archived_doc:
send_to(archived_doc, doc.get('desk'))
new_guid = superdesk.get_resource_service('archive').duplicate_content(archived_doc)
new_guids.append(new_guid)
else:
msg = 'Fail to found ingest item with guid: %s' % doc.get('guid')
raise SuperdeskApiError.notFoundError(msg)
else:
# We are fetching from ingest
if not is_workflow_state_transition_valid('fetch_as_from_ingest', ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
archived = utcnow()
superdesk.get_resource_service('ingest').patch(ingest_doc.get('_id'), {'archived': archived})
doc['archived'] = archived
dest_doc = dict(ingest_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, doc.get('desk'))
dest_doc[config.CONTENT_STATE] = STATE_FETCHED
dest_doc[INGEST_ID] = ingest_doc['_id']
dest_doc[FAMILY_ID] = ingest_doc['_id']
remove_unwanted(dest_doc)
for ref in [ref for group in dest_doc.get('groups', [])
for ref in group.get('refs', []) if 'residRef' in ref]:
ref['location'] = ARCHIVE
ref['guid'] = ref['residRef']
set_original_creator(dest_doc)
if doc.get(PACKAGE):
links = dest_doc.get(LINKED_IN_PACKAGES, [])
links.append({PACKAGE: doc.get(PACKAGE)})
dest_doc[LINKED_IN_PACKAGES] = links
desk = doc.get('desk')
refs = [{'guid': ref.get('residRef'), 'desk': desk, PACKAGE: dest_doc.get('_id')}
for group in dest_doc.get('groups', [])
for ref in group.get('refs', []) if 'residRef' in ref]
if refs:
new_ref_guids = self.create(refs)
count = 0
for ref in [ref for group in dest_doc.get('groups', [])
for ref in group.get('refs', []) if 'residRef' in ref]:
ref['residRef'] = ref['guid'] = new_ref_guids[count]
count += 1
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('guid'))
push_notification('item:fetch', item=str(ingest_doc.get('_id')))
return new_guids