def create(self, docs, **kwargs): new_guids = [] provider = get_resource_service('ingest_providers').find_one(source='aapmm', req=None) for doc in docs: if not doc.get('desk'): # if no desk is selected then it is bad request raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.") archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid']) dest_doc = dict(archived_doc) new_id = generate_guid(type=GUID_TAG) new_guids.append(new_id) dest_doc['_id'] = new_id generate_unique_id_and_name(dest_doc) if provider: dest_doc['ingest_provider'] = str(provider[superdesk.config.ID_FIELD]) dest_doc[config.VERSION] = 1 send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage')) dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED) dest_doc[INGEST_ID] = archived_doc['_id'] dest_doc[FAMILY_ID] = archived_doc['_id'] remove_unwanted(dest_doc) set_original_creator(dest_doc) superdesk.get_resource_service(ARCHIVE).post([dest_doc]) insert_into_versions(dest_doc.get('_id')) return new_guids
def _create_image_items(self, image_links, text_item): """Create a list of picture items that represent the external images located on given URLs. Each created item's `firstcreated` and `versioncreated` fields are set to the same value as the values of these fields in `text_item`. :param iterable image_links: list of image URLs :param dict text_item: the "main" text item the images are related to :return: list of created image items (as dicts) """ image_items = [] for image_url in image_links: img_item = { 'guid': generate_guid(type=GUID_TAG), 'type': 'picture', 'firstcreated': text_item.get('firstcreated'), 'versioncreated': text_item.get('versioncreated'), 'renditions': { 'baseImage': { 'href': image_url } } } image_items.append(img_item) return image_items
def create(self, docs, **kwargs): new_guids = [] provider = get_resource_service('ingest_providers').find_one( source='aapmm', req=None) for doc in docs: if not doc.get('desk'): # if no desk is selected then it is bad request raise SuperdeskApiError.badRequestError( "Destination desk cannot be empty.") archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid']) dest_doc = dict(archived_doc) new_id = generate_guid(type=GUID_TAG) new_guids.append(new_id) dest_doc['_id'] = new_id generate_unique_id_and_name(dest_doc) if provider: dest_doc['ingest_provider'] = str( provider[superdesk.config.ID_FIELD]) dest_doc[config.VERSION] = 1 send_to(dest_doc, doc.get('desk'), doc.get('stage')) dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED) dest_doc[INGEST_ID] = archived_doc['_id'] dest_doc[FAMILY_ID] = archived_doc['_id'] remove_unwanted(dest_doc) set_original_creator(dest_doc) superdesk.get_resource_service(ARCHIVE).post([dest_doc]) insert_into_versions(dest_doc.get('_id')) return new_guids
def fetch(self, docs, id=None, **kwargs): id_of_fetched_items = [] for doc in docs: id_of_item_to_be_fetched = doc.get('_id') if id is None else id desk_id = doc.get('desk') stage_id = doc.get('stage') ingest_service = get_resource_service('ingest') ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched) if not ingest_doc: raise SuperdeskApiError.notFoundError( 'Fail to found ingest item with _id: %s' % id_of_item_to_be_fetched) if not is_workflow_state_transition_valid( 'fetch_from_ingest', ingest_doc[config.CONTENT_STATE]): raise InvalidStateTransitionError() if doc.get('macro'): # there is a macro so transform it ingest_doc = get_resource_service('macros').execute_macro( ingest_doc, doc.get('macro')) archived = utcnow() ingest_service.patch(id_of_item_to_be_fetched, {'archived': archived}) dest_doc = dict(ingest_doc) new_id = generate_guid(type=GUID_TAG) id_of_fetched_items.append(new_id) dest_doc['_id'] = new_id dest_doc['guid'] = new_id dest_doc['destination_groups'] = doc.get('destination_groups') generate_unique_id_and_name(dest_doc) dest_doc[config.VERSION] = 1 send_to(dest_doc, desk_id, stage_id) dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED) dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id'] remove_unwanted(dest_doc) set_original_creator(dest_doc) self.__fetch_items_in_package(dest_doc, desk_id, stage_id, doc.get('state', STATE_FETCHED), doc.get('destination_groups')) get_resource_service(ARCHIVE).post([dest_doc]) insert_into_versions(doc=dest_doc) build_custom_hateoas(custom_hateoas, dest_doc) doc.update(dest_doc) if kwargs.get('notify', True): push_notification('item:fetch', fetched=1) return id_of_fetched_items
def ingest_item(item, provider, rule_set=None, routing_scheme=None): try: item.setdefault(superdesk.config.ID_FIELD, generate_guid(type=GUID_NEWSML)) item[FAMILY_ID] = item[superdesk.config.ID_FIELD] providers[provider.get("type")].provider = provider item["ingest_provider"] = str(provider[superdesk.config.ID_FIELD]) item.setdefault("source", provider.get("source", "")) set_default_state(item, STATE_INGESTED) item["expiry"] = get_expiry_date( provider.get("content_expiry", INGEST_EXPIRY_MINUTES), item.get("versioncreated") ) if "anpa-category" in item: process_anpa_category(item, provider) if "subject" in item: process_iptc_codes(item, provider) apply_rule_set(item, provider, rule_set) ingest_service = superdesk.get_resource_service("ingest") if item.get("ingest_provider_sequence") is None: ingest_service.set_ingest_provider_sequence(item, provider) rend = item.get("renditions", {}) if rend: baseImageRend = rend.get("baseImage") or next(iter(rend.values())) if baseImageRend: href = providers[provider.get("type")].prepare_href(baseImageRend["href"]) update_renditions(item, href) old_item = ingest_service.find_one(guid=item[GUID_FIELD], req=None) if old_item: # In case we already have the item, preserve the _id item[superdesk.config.ID_FIELD] = old_item[superdesk.config.ID_FIELD] ingest_service.put_in_mongo(item[superdesk.config.ID_FIELD], item) else: try: ingest_service.post_in_mongo([item]) except HTTPException as e: logger.error("Exception while persisting item in ingest collection", e) if routing_scheme: routed = ingest_service.find_one(_id=item[superdesk.config.ID_FIELD], req=None) superdesk.get_resource_service("routing_schemes").apply_routing_scheme(routed, provider, routing_scheme) except Exception as ex: logger.exception(ex) try: superdesk.app.sentry.captureException() except: pass return False return True
def fetch(self, docs, id=None, **kwargs): id_of_fetched_items = [] for doc in docs: id_of_item_to_be_fetched = doc.get('_id') if id is None else id desk_id = doc.get('desk') stage_id = doc.get('stage') ingest_service = get_resource_service('ingest') ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched) if not ingest_doc: raise SuperdeskApiError.notFoundError('Fail to found ingest item with _id: %s' % id_of_item_to_be_fetched) if not is_workflow_state_transition_valid('fetch_from_ingest', ingest_doc[config.CONTENT_STATE]): raise InvalidStateTransitionError() if doc.get('macro'): # there is a macro so transform it ingest_doc = get_resource_service('macros').execute_macro(ingest_doc, doc.get('macro')) archived = utcnow() ingest_service.patch(id_of_item_to_be_fetched, {'archived': archived}) dest_doc = dict(ingest_doc) new_id = generate_guid(type=GUID_TAG) id_of_fetched_items.append(new_id) dest_doc['_id'] = new_id dest_doc['guid'] = new_id dest_doc['destination_groups'] = doc.get('destination_groups') generate_unique_id_and_name(dest_doc) dest_doc[config.VERSION] = 1 send_to(dest_doc, desk_id, stage_id) dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED) dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id'] remove_unwanted(dest_doc) set_original_creator(dest_doc) self.__fetch_items_in_package(dest_doc, desk_id, stage_id, doc.get('state', STATE_FETCHED), doc.get('destination_groups')) get_resource_service(ARCHIVE).post([dest_doc]) insert_into_versions(doc=dest_doc) build_custom_hateoas(custom_hateoas, dest_doc) doc.update(dest_doc) if kwargs.get('notify', True): push_notification('item:fetch', fetched=1) return id_of_fetched_items
def on_create(self, docs): for doc in docs: update_dates_for(doc) doc['original_creator'] = str(get_user().get('_id')) doc['guid'] = generate_guid(type=GUID_TAG) # set the blog_preferences by merging given preferences with global_prefs global_prefs = get_resource_service('global_preferences').get_global_prefs() prefs = global_prefs.copy() prefs.update(doc.get('blog_preferences', {})) doc['blog_preferences'] = prefs # save a snapshot of the theme in the `theme` field if 'theme' in prefs: doc['theme'] = self.get_theme_snapshot(prefs['theme'])
def _create_package(self, text_item, image_items): """Create a new content package from given content items. The package's `main` group contains only the references to given items, not the items themselves. In the list of references, the reference to the text item preceeds the references to image items. Package's `firstcreated` and `versioncreated` fields are set to values of these fields in `text_item`. :param dict text_item: item representing the text content :param list image_items: list of items (dicts) representing the images related to the text content :return: the created content package :rtype: dict """ package = { 'type': 'composite', 'guid': generate_guid(type=GUID_TAG), 'firstcreated': text_item['firstcreated'], 'versioncreated': text_item['versioncreated'], 'groups': [ { 'id': 'root', 'role': 'grpRole:NEP', 'refs': [{'idRef': 'main'}], }, { 'id': 'main', 'role': 'main', 'refs': [], } ] } item_references = package['groups'][1]['refs'] item_references.append({'residRef': text_item['guid']}) for image in image_items: item_references.append({'residRef': image['guid']}) return package
def ingest_items_for(self, desk, no_of_stories, skip_index): desk_id = desk['_id'] stage_id = desk['incoming_stage'] bucket_size = min(100, no_of_stories) no_of_buckets = len(range(0, no_of_stories, bucket_size)) for x in range(0, no_of_buckets): skip = x * bucket_size * skip_index self.logger.info('Page : {}, skip: {}'.format(x + 1, skip)) cursor = get_resource_service('text_archive').get_from_mongo(None, {}) cursor.skip(skip) cursor.limit(bucket_size) items = list(cursor) self.logger.info('Inserting {} items'.format(len(items))) archive_items = [] for item in items: dest_doc = dict(item) new_id = generate_guid(type=GUID_TAG) dest_doc['_id'] = new_id dest_doc['guid'] = new_id generate_unique_id_and_name(dest_doc) dest_doc[app.config['VERSION']] = 1 dest_doc['state'] = 'fetched' user_id = desk.get('members', [{'user': None}])[0].get('user') dest_doc['original_creator'] = user_id dest_doc['version_creator'] = user_id send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id) dest_doc[FAMILY_ID] = item['_id'] remove_unwanted(dest_doc) archive_items.append(dest_doc) get_resource_service(ARCHIVE).post(archive_items) for item in archive_items: insert_into_versions(id_=item['_id'])
def on_create(self, docs): for doc in docs: update_dates_for(doc) doc['original_creator'] = str(get_user().get('_id')) doc['guid'] = generate_guid(type=GUID_TAG)
def ingest_item(item, provider, rule_set=None, routing_scheme=None): try: item.setdefault(superdesk.config.ID_FIELD, generate_guid(type=GUID_NEWSML)) item[FAMILY_ID] = item[superdesk.config.ID_FIELD] providers[provider.get('type')].provider = provider item['ingest_provider'] = str(provider[superdesk.config.ID_FIELD]) item.setdefault('source', provider.get('source', '')) set_default_state(item, STATE_INGESTED) item['expiry'] = get_expiry_date( provider.get('content_expiry', INGEST_EXPIRY_MINUTES), item.get('versioncreated')) if 'anpa-category' in item: process_anpa_category(item, provider) if 'subject' in item: process_iptc_codes(item, provider) apply_rule_set(item, provider, rule_set) ingest_service = superdesk.get_resource_service('ingest') if item.get('ingest_provider_sequence') is None: ingest_service.set_ingest_provider_sequence(item, provider) rend = item.get('renditions', {}) if rend: baseImageRend = rend.get('baseImage') or next(iter(rend.values())) if baseImageRend: href = providers[provider.get('type')].prepare_href( baseImageRend['href']) update_renditions(item, href) old_item = ingest_service.find_one(guid=item[GUID_FIELD], req=None) if old_item: # In case we already have the item, preserve the _id item[superdesk.config.ID_FIELD] = old_item[ superdesk.config.ID_FIELD] ingest_service.put_in_mongo(item[superdesk.config.ID_FIELD], item) else: try: ingest_service.post_in_mongo([item]) except HTTPException as e: logger.error( "Exception while persisting item in ingest collection", e) if routing_scheme: routed = ingest_service.find_one( _id=item[superdesk.config.ID_FIELD], req=None) superdesk.get_resource_service( 'routing_schemes').apply_routing_scheme( routed, provider, routing_scheme) except Exception as ex: logger.exception(ex) try: superdesk.app.sentry.captureException() except: pass return False return True