def create(self, docs, **kwargs):
new_guids = []
provider = get_resource_service('ingest_providers').find_one(source='aapmm', req=None)
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
return new_guids
def _create_image_items(self, image_links, text_item):
"""Create a list of picture items that represent the external images
located on given URLs.
Each created item's `firstcreated` and `versioncreated` fields are set
to the same value as the values of these fields in `text_item`.
:param iterable image_links: list of image URLs
:param dict text_item: the "main" text item the images are related to
:return: list of created image items (as dicts)
"""
image_items = []
for image_url in image_links:
img_item = {
'guid': generate_guid(type=GUID_TAG),
'type': 'picture',
'firstcreated': text_item.get('firstcreated'),
'versioncreated': text_item.get('versioncreated'),
'renditions': {
'baseImage': {
'href': image_url
}
}
}
image_items.append(img_item)
return image_items
def create(self, docs, **kwargs):
new_guids = []
provider = get_resource_service('ingest_providers').find_one(
source='aapmm', req=None)
for doc in docs:
if not doc.get('desk'):
# if no desk is selected then it is bad request
raise SuperdeskApiError.badRequestError(
"Destination desk cannot be empty.")
archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])
dest_doc = dict(archived_doc)
new_id = generate_guid(type=GUID_TAG)
new_guids.append(new_id)
dest_doc['_id'] = new_id
generate_unique_id_and_name(dest_doc)
if provider:
dest_doc['ingest_provider'] = str(
provider[superdesk.config.ID_FIELD])
dest_doc[config.VERSION] = 1
send_to(dest_doc, doc.get('desk'), doc.get('stage'))
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = archived_doc['_id']
dest_doc[FAMILY_ID] = archived_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
superdesk.get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(dest_doc.get('_id'))
return new_guids
def fetch(self, docs, id=None, **kwargs):
id_of_fetched_items = []
for doc in docs:
id_of_item_to_be_fetched = doc.get('_id') if id is None else id
desk_id = doc.get('desk')
stage_id = doc.get('stage')
ingest_service = get_resource_service('ingest')
ingest_doc = ingest_service.find_one(req=None,
_id=id_of_item_to_be_fetched)
if not ingest_doc:
raise SuperdeskApiError.notFoundError(
'Fail to found ingest item with _id: %s' %
id_of_item_to_be_fetched)
if not is_workflow_state_transition_valid(
'fetch_from_ingest', ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
if doc.get('macro'): # there is a macro so transform it
ingest_doc = get_resource_service('macros').execute_macro(
ingest_doc, doc.get('macro'))
archived = utcnow()
ingest_service.patch(id_of_item_to_be_fetched,
{'archived': archived})
dest_doc = dict(ingest_doc)
new_id = generate_guid(type=GUID_TAG)
id_of_fetched_items.append(new_id)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
dest_doc['destination_groups'] = doc.get('destination_groups')
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, desk_id, stage_id)
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
self.__fetch_items_in_package(dest_doc, desk_id, stage_id,
doc.get('state', STATE_FETCHED),
doc.get('destination_groups'))
get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(doc=dest_doc)
build_custom_hateoas(custom_hateoas, dest_doc)
doc.update(dest_doc)
if kwargs.get('notify', True):
push_notification('item:fetch', fetched=1)
return id_of_fetched_items
def ingest_item(item, provider, rule_set=None, routing_scheme=None):
try:
item.setdefault(superdesk.config.ID_FIELD, generate_guid(type=GUID_NEWSML))
item[FAMILY_ID] = item[superdesk.config.ID_FIELD]
providers[provider.get("type")].provider = provider
item["ingest_provider"] = str(provider[superdesk.config.ID_FIELD])
item.setdefault("source", provider.get("source", ""))
set_default_state(item, STATE_INGESTED)
item["expiry"] = get_expiry_date(
provider.get("content_expiry", INGEST_EXPIRY_MINUTES), item.get("versioncreated")
)
if "anpa-category" in item:
process_anpa_category(item, provider)
if "subject" in item:
process_iptc_codes(item, provider)
apply_rule_set(item, provider, rule_set)
ingest_service = superdesk.get_resource_service("ingest")
if item.get("ingest_provider_sequence") is None:
ingest_service.set_ingest_provider_sequence(item, provider)
rend = item.get("renditions", {})
if rend:
baseImageRend = rend.get("baseImage") or next(iter(rend.values()))
if baseImageRend:
href = providers[provider.get("type")].prepare_href(baseImageRend["href"])
update_renditions(item, href)
old_item = ingest_service.find_one(guid=item[GUID_FIELD], req=None)
if old_item:
# In case we already have the item, preserve the _id
item[superdesk.config.ID_FIELD] = old_item[superdesk.config.ID_FIELD]
ingest_service.put_in_mongo(item[superdesk.config.ID_FIELD], item)
else:
try:
ingest_service.post_in_mongo([item])
except HTTPException as e:
logger.error("Exception while persisting item in ingest collection", e)
if routing_scheme:
routed = ingest_service.find_one(_id=item[superdesk.config.ID_FIELD], req=None)
superdesk.get_resource_service("routing_schemes").apply_routing_scheme(routed, provider, routing_scheme)
except Exception as ex:
logger.exception(ex)
try:
superdesk.app.sentry.captureException()
except:
pass
return False
return True
def fetch(self, docs, id=None, **kwargs):
id_of_fetched_items = []
for doc in docs:
id_of_item_to_be_fetched = doc.get('_id') if id is None else id
desk_id = doc.get('desk')
stage_id = doc.get('stage')
ingest_service = get_resource_service('ingest')
ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched)
if not ingest_doc:
raise SuperdeskApiError.notFoundError('Fail to found ingest item with _id: %s' %
id_of_item_to_be_fetched)
if not is_workflow_state_transition_valid('fetch_from_ingest', ingest_doc[config.CONTENT_STATE]):
raise InvalidStateTransitionError()
if doc.get('macro'): # there is a macro so transform it
ingest_doc = get_resource_service('macros').execute_macro(ingest_doc, doc.get('macro'))
archived = utcnow()
ingest_service.patch(id_of_item_to_be_fetched, {'archived': archived})
dest_doc = dict(ingest_doc)
new_id = generate_guid(type=GUID_TAG)
id_of_fetched_items.append(new_id)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
dest_doc['destination_groups'] = doc.get('destination_groups')
generate_unique_id_and_name(dest_doc)
dest_doc[config.VERSION] = 1
send_to(dest_doc, desk_id, stage_id)
dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id']
remove_unwanted(dest_doc)
set_original_creator(dest_doc)
self.__fetch_items_in_package(dest_doc, desk_id, stage_id,
doc.get('state', STATE_FETCHED),
doc.get('destination_groups'))
get_resource_service(ARCHIVE).post([dest_doc])
insert_into_versions(doc=dest_doc)
build_custom_hateoas(custom_hateoas, dest_doc)
doc.update(dest_doc)
if kwargs.get('notify', True):
push_notification('item:fetch', fetched=1)
return id_of_fetched_items
def on_create(self, docs):
for doc in docs:
update_dates_for(doc)
doc['original_creator'] = str(get_user().get('_id'))
doc['guid'] = generate_guid(type=GUID_TAG)
# set the blog_preferences by merging given preferences with global_prefs
global_prefs = get_resource_service('global_preferences').get_global_prefs()
prefs = global_prefs.copy()
prefs.update(doc.get('blog_preferences', {}))
doc['blog_preferences'] = prefs
# save a snapshot of the theme in the `theme` field
if 'theme' in prefs:
doc['theme'] = self.get_theme_snapshot(prefs['theme'])
def _create_package(self, text_item, image_items):
"""Create a new content package from given content items.
The package's `main` group contains only the references to given items,
not the items themselves. In the list of references, the reference to
the text item preceeds the references to image items.
Package's `firstcreated` and `versioncreated` fields are set to values
of these fields in `text_item`.
:param dict text_item: item representing the text content
:param list image_items: list of items (dicts) representing the images
related to the text content
:return: the created content package
:rtype: dict
"""
package = {
'type': 'composite',
'guid': generate_guid(type=GUID_TAG),
'firstcreated': text_item['firstcreated'],
'versioncreated': text_item['versioncreated'],
'groups': [
{
'id': 'root',
'role': 'grpRole:NEP',
'refs': [{'idRef': 'main'}],
}, {
'id': 'main',
'role': 'main',
'refs': [],
}
]
}
item_references = package['groups'][1]['refs']
item_references.append({'residRef': text_item['guid']})
for image in image_items:
item_references.append({'residRef': image['guid']})
return package
def ingest_items_for(self, desk, no_of_stories, skip_index):
desk_id = desk['_id']
stage_id = desk['incoming_stage']
bucket_size = min(100, no_of_stories)
no_of_buckets = len(range(0, no_of_stories, bucket_size))
for x in range(0, no_of_buckets):
skip = x * bucket_size * skip_index
self.logger.info('Page : {}, skip: {}'.format(x + 1, skip))
cursor = get_resource_service('text_archive').get_from_mongo(None, {})
cursor.skip(skip)
cursor.limit(bucket_size)
items = list(cursor)
self.logger.info('Inserting {} items'.format(len(items)))
archive_items = []
for item in items:
dest_doc = dict(item)
new_id = generate_guid(type=GUID_TAG)
dest_doc['_id'] = new_id
dest_doc['guid'] = new_id
generate_unique_id_and_name(dest_doc)
dest_doc[app.config['VERSION']] = 1
dest_doc['state'] = 'fetched'
user_id = desk.get('members', [{'user': None}])[0].get('user')
dest_doc['original_creator'] = user_id
dest_doc['version_creator'] = user_id
send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id)
dest_doc[FAMILY_ID] = item['_id']
remove_unwanted(dest_doc)
archive_items.append(dest_doc)
get_resource_service(ARCHIVE).post(archive_items)
for item in archive_items:
insert_into_versions(id_=item['_id'])
def on_create(self, docs):
for doc in docs:
update_dates_for(doc)
doc['original_creator'] = str(get_user().get('_id'))
doc['guid'] = generate_guid(type=GUID_TAG)
def ingest_item(item, provider, rule_set=None, routing_scheme=None):
try:
item.setdefault(superdesk.config.ID_FIELD,
generate_guid(type=GUID_NEWSML))
item[FAMILY_ID] = item[superdesk.config.ID_FIELD]
providers[provider.get('type')].provider = provider
item['ingest_provider'] = str(provider[superdesk.config.ID_FIELD])
item.setdefault('source', provider.get('source', ''))
set_default_state(item, STATE_INGESTED)
item['expiry'] = get_expiry_date(
provider.get('content_expiry', INGEST_EXPIRY_MINUTES),
item.get('versioncreated'))
if 'anpa-category' in item:
process_anpa_category(item, provider)
if 'subject' in item:
process_iptc_codes(item, provider)
apply_rule_set(item, provider, rule_set)
ingest_service = superdesk.get_resource_service('ingest')
if item.get('ingest_provider_sequence') is None:
ingest_service.set_ingest_provider_sequence(item, provider)
rend = item.get('renditions', {})
if rend:
baseImageRend = rend.get('baseImage') or next(iter(rend.values()))
if baseImageRend:
href = providers[provider.get('type')].prepare_href(
baseImageRend['href'])
update_renditions(item, href)
old_item = ingest_service.find_one(guid=item[GUID_FIELD], req=None)
if old_item:
# In case we already have the item, preserve the _id
item[superdesk.config.ID_FIELD] = old_item[
superdesk.config.ID_FIELD]
ingest_service.put_in_mongo(item[superdesk.config.ID_FIELD], item)
else:
try:
ingest_service.post_in_mongo([item])
except HTTPException as e:
logger.error(
"Exception while persisting item in ingest collection", e)
if routing_scheme:
routed = ingest_service.find_one(
_id=item[superdesk.config.ID_FIELD], req=None)
superdesk.get_resource_service(
'routing_schemes').apply_routing_scheme(
routed, provider, routing_scheme)
except Exception as ex:
logger.exception(ex)
try:
superdesk.app.sentry.captureException()
except:
pass
return False
return True
