예제 #1
0
    def create(self, docs, **kwargs):
        new_guids = []
        provider = get_resource_service('ingest_providers').find_one(
            source='aapmm', req=None)
        for doc in docs:
            if not doc.get('desk'):
                # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError(
                    "Destination desk cannot be empty.")

            archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc['_id'] = new_id
            generate_unique_id_and_name(dest_doc)

            if provider:
                dest_doc['ingest_provider'] = str(
                    provider[superdesk.config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(dest_doc, doc.get('desk'), doc.get('stage'))
            dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
            dest_doc[INGEST_ID] = archived_doc['_id']
            dest_doc[FAMILY_ID] = archived_doc['_id']
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc.get('_id'))

        return new_guids
예제 #2
0
    def create(self, docs, **kwargs):
        new_guids = []
        provider = self.get_provider()
        for doc in docs:
            if not doc.get('desk'):
                # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
            try:
                archived_doc = self.fetch(doc['guid'])
            except FileNotFoundError as ex:
                raise ProviderError.externalProviderError(ex, provider)

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc['_id'] = new_id
            generate_unique_id_and_name(dest_doc)

            if provider:
                dest_doc['ingest_provider'] = str(provider[superdesk.config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = archived_doc['_id']
            dest_doc[FAMILY_ID] = archived_doc['_id']
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc.get('_id'))

        return new_guids
예제 #3
0
    def create(self, docs, **kwargs):
        new_guids = []
        provider = get_resource_service("ingest_providers").find_one(source="aapmm", req=None)
        if provider and "config" in provider and "username" in provider["config"]:
            self.backend.set_credentials(provider["config"]["username"], provider["config"]["password"])
        for doc in docs:
            if not doc.get("desk"):
                # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")
            try:
                archived_doc = self.backend.find_one_raw(doc["guid"], doc["guid"])
            except FileNotFoundError as ex:
                raise ProviderError.externalProviderError(ex, provider)

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc["_id"] = new_id
            generate_unique_id_and_name(dest_doc)

            if provider:
                dest_doc["ingest_provider"] = str(provider[superdesk.config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc, update=None, desk_id=doc.get("desk"), stage_id=doc.get("stage"))
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = archived_doc["_id"]
            dest_doc[FAMILY_ID] = archived_doc["_id"]
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc.get("_id"))

        return new_guids
예제 #4
0
    def create(self, docs, **kwargs):
        new_guids = []
        provider = get_resource_service('ingest_providers').find_one(source='aapmm', req=None)
        for doc in docs:
            if not doc.get('desk'):
                # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")

            archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc['_id'] = new_id
            generate_unique_id_and_name(dest_doc)

            if provider:
                dest_doc['ingest_provider'] = str(provider[superdesk.config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
            dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
            dest_doc[INGEST_ID] = archived_doc['_id']
            dest_doc[FAMILY_ID] = archived_doc['_id']
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc.get('_id'))

        return new_guids
예제 #5
0
    def fetch(self, docs, id=None, **kwargs):
        id_of_fetched_items = []

        for doc in docs:
            id_of_item_to_be_fetched = doc.get('_id') if id is None else id

            desk_id = doc.get('desk')
            stage_id = doc.get('stage')

            ingest_service = get_resource_service('ingest')
            ingest_doc = ingest_service.find_one(req=None,
                                                 _id=id_of_item_to_be_fetched)

            if not ingest_doc:
                raise SuperdeskApiError.notFoundError(
                    'Fail to found ingest item with _id: %s' %
                    id_of_item_to_be_fetched)

            if not is_workflow_state_transition_valid(
                    'fetch_from_ingest', ingest_doc[config.CONTENT_STATE]):
                raise InvalidStateTransitionError()

            if doc.get('macro'):  # there is a macro so transform it
                ingest_doc = get_resource_service('macros').execute_macro(
                    ingest_doc, doc.get('macro'))

            archived = utcnow()
            ingest_service.patch(id_of_item_to_be_fetched,
                                 {'archived': archived})

            dest_doc = dict(ingest_doc)
            new_id = generate_guid(type=GUID_TAG)
            id_of_fetched_items.append(new_id)
            dest_doc['_id'] = new_id
            dest_doc['guid'] = new_id
            dest_doc['destination_groups'] = doc.get('destination_groups')
            generate_unique_id_and_name(dest_doc)

            dest_doc[config.VERSION] = 1
            send_to(dest_doc, desk_id, stage_id)
            dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
            dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id']

            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)
            self.__fetch_items_in_package(dest_doc, desk_id, stage_id,
                                          doc.get('state', STATE_FETCHED),
                                          doc.get('destination_groups'))

            get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(doc=dest_doc)
            build_custom_hateoas(custom_hateoas, dest_doc)
            doc.update(dest_doc)

        if kwargs.get('notify', True):
            push_notification('item:fetch', fetched=1)

        return id_of_fetched_items
예제 #6
0
    def _publish_associated_items(self, original, updates={}, publish=False):
        """Refresh and publish associated items before publishing. The publishing is done if the setting
        PUBLISH_ASSOCIATED_ITEMS was true.

        Any further updates made to basic metadata done after item was associated will be carried on and
        used when validating those items.
        """
        publish_service = None
        if config.PUBLISH_ASSOCIATED_ITEMS and publish_services.get(
                self.publish_type):
            publish_service = get_resource_service(
                publish_services[self.publish_type])

        associations = original.get(ASSOCIATIONS) or {}
        for associations_key, item in associations.items():
            if type(item) == dict and item.get(config.ID_FIELD):
                keys = [
                    key for key in DEFAULT_SCHEMA.keys()
                    if key not in PRESERVED_FIELDS
                ]

                if app.settings.get('COPY_METADATA_FROM_PARENT') and item.get(
                        ITEM_TYPE) in MEDIA_TYPES:
                    original_item = original
                    keys = FIELDS_TO_COPY_FOR_ASSOCIATED_ITEM
                else:
                    original_item = super().find_one(
                        req=None, _id=item[config.ID_FIELD]) or {}

                update_item_data(item, original_item, keys)
                remove_unwanted(item)

                if publish_service and publish and item['type'] in MEDIA_TYPES:
                    if item.get('task', {}).get('stage', None):
                        del item['task']['stage']
                    if item['state'] not in PUBLISH_STATES:
                        get_resource_service('archive_publish').patch(
                            id=item.pop(config.ID_FIELD), updates=item)
                    else:
                        publish_service.patch(id=item.pop(config.ID_FIELD),
                                              updates=item)
                    item['state'] = self.published_state
                    original[ASSOCIATIONS][associations_key][
                        'state'] = self.published_state
                    original[ASSOCIATIONS][associations_key][
                        'operation'] = self.publish_type
                    if ASSOCIATIONS not in updates:
                        updates[ASSOCIATIONS] = original[ASSOCIATIONS]
                    updates[ASSOCIATIONS][associations_key] = original[
                        ASSOCIATIONS][associations_key]
                elif publish:
                    # Publishing an item with an association set the state and operation of the embedded association
                    original[ASSOCIATIONS][associations_key][
                        'state'] = self.published_state
                    original[ASSOCIATIONS][associations_key][
                        'operation'] = self.publish_type
예제 #7
0
    def create(self, docs, **kwargs):
        search_provider = get_resource_service('search_providers').find_one(
            search_provider=PROVIDER_NAME, req=None)

        if not search_provider or search_provider.get('is_closed', False):
            raise SuperdeskApiError.badRequestError(
                'No search provider found or the search provider is closed.')

        if 'config' in search_provider:
            self.backend.set_credentials(search_provider['config'])

        new_guids = []
        for doc in docs:
            if not doc.get(
                    'desk'):  # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError(
                    "Destination desk cannot be empty.")

            try:
                archived_doc = self.backend.find_one_raw(
                    doc['guid'], doc['guid'])
            except FileNotFoundError as ex:
                raise ProviderError.externalProviderError(ex, search_provider)

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc[config.ID_FIELD] = new_id
            generate_unique_id_and_name(dest_doc)

            if search_provider:
                dest_doc['ingest_provider'] = str(
                    search_provider[config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc,
                    update=None,
                    desk_id=doc.get('desk'),
                    stage_id=doc.get('stage'))
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = archived_doc[config.ID_FIELD]
            dest_doc[FAMILY_ID] = archived_doc[config.ID_FIELD]
            dest_doc[ITEM_OPERATION] = ITEM_FETCH
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc[config.ID_FIELD])

            get_resource_service('search_providers').system_update(
                search_provider[config.ID_FIELD],
                {'last_item_update': utcnow()}, search_provider)

        return new_guids
예제 #8
0
    def fetch(self, docs, id=None, **kwargs):
        id_of_fetched_items = []

        for doc in docs:
            id_of_item_to_be_fetched = doc.get('_id') if id is None else id

            desk_id = doc.get('desk')
            stage_id = doc.get('stage')

            ingest_service = get_resource_service('ingest')
            ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched)

            if not ingest_doc:
                raise SuperdeskApiError.notFoundError('Fail to found ingest item with _id: %s' %
                                                      id_of_item_to_be_fetched)

            if not is_workflow_state_transition_valid('fetch_from_ingest', ingest_doc[config.CONTENT_STATE]):
                raise InvalidStateTransitionError()

            if doc.get('macro'):  # there is a macro so transform it
                ingest_doc = get_resource_service('macros').execute_macro(ingest_doc, doc.get('macro'))

            archived = utcnow()
            ingest_service.patch(id_of_item_to_be_fetched, {'archived': archived})

            dest_doc = dict(ingest_doc)
            new_id = generate_guid(type=GUID_TAG)
            id_of_fetched_items.append(new_id)
            dest_doc['_id'] = new_id
            dest_doc['guid'] = new_id
            dest_doc['destination_groups'] = doc.get('destination_groups')
            generate_unique_id_and_name(dest_doc)

            dest_doc[config.VERSION] = 1
            send_to(dest_doc, desk_id, stage_id)
            dest_doc[config.CONTENT_STATE] = doc.get('state', STATE_FETCHED)
            dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc['_id']

            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)
            self.__fetch_items_in_package(dest_doc, desk_id, stage_id,
                                          doc.get('state', STATE_FETCHED),
                                          doc.get('destination_groups'))

            get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(doc=dest_doc)
            build_custom_hateoas(custom_hateoas, dest_doc)
            doc.update(dest_doc)

        if kwargs.get('notify', True):
            push_notification('item:fetch', fetched=1)

        return id_of_fetched_items
예제 #9
0
    def fetch(self, docs, id=None, **kwargs):
        id_of_fetched_items = []

        for doc in docs:
            id_of_item_to_be_fetched = doc.get("_id") if id is None else id

            desk_id = doc.get("desk")
            stage_id = doc.get("stage")

            ingest_service = get_resource_service("ingest")
            ingest_doc = ingest_service.find_one(req=None, _id=id_of_item_to_be_fetched)

            if not ingest_doc:
                raise SuperdeskApiError.notFoundError(
                    "Fail to found ingest item with _id: %s" % id_of_item_to_be_fetched
                )

            if not is_workflow_state_transition_valid("fetch_from_ingest", ingest_doc[ITEM_STATE]):
                raise InvalidStateTransitionError()

            if doc.get("macro"):  # there is a macro so transform it
                ingest_doc = get_resource_service("macros").execute_macro(ingest_doc, doc.get("macro"))

            archived = utcnow()
            ingest_service.patch(id_of_item_to_be_fetched, {"archived": archived})

            dest_doc = dict(ingest_doc)
            new_id = generate_guid(type=GUID_TAG)
            id_of_fetched_items.append(new_id)
            dest_doc["_id"] = new_id
            dest_doc["guid"] = new_id
            generate_unique_id_and_name(dest_doc)

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc, desk_id=desk_id, stage_id=stage_id)
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc["_id"]
            dest_doc[ITEM_OPERATION] = ITEM_FETCH

            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)
            self.__fetch_items_in_package(dest_doc, desk_id, stage_id, doc.get(ITEM_STATE, CONTENT_STATE.FETCHED))

            get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(doc=dest_doc)
            build_custom_hateoas(custom_hateoas, dest_doc)
            doc.update(dest_doc)

        if kwargs.get("notify", True):
            push_notification("item:fetch", fetched=1)

        return id_of_fetched_items
예제 #10
0
def _update_archive(guid, item):
    """
    Assigns a State to the content, removes unwanted attributes, sets the original creator and updates the item in
    archive collection.

    :param guid:
    :param item: from ingest collection
    """

    item[config.CONTENT_STATE] = STATE_FETCHED
    remove_unwanted(item)
    set_original_creator(item)
    superdesk.get_resource_service(ARCHIVE).update(guid, item)
예제 #11
0
    def create(self, docs, **kwargs):
        for doc in docs:
            ingest_doc = superdesk.get_resource_service("ingest").find_one(req=None, _id=doc.get("guid"))
            if not ingest_doc:
                msg = "Fail to found ingest item with guid: %s" % doc.get("guid")
                raise SuperdeskApiError.notFoundError(msg)

            if not is_workflow_state_transition_valid("fetch_as_from_ingest", ingest_doc[config.CONTENT_STATE]):
                raise InvalidStateTransitionError()

            archived = utcnow()
            superdesk.get_resource_service("ingest").patch(ingest_doc.get("_id"), {"archived": archived})
            doc["archived"] = archived

            archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, _id=doc.get("guid"))
            if not archived_doc:
                dest_doc = dict(ingest_doc)
                dest_doc[config.VERSION] = 1
                send_to(dest_doc, doc.get("desk"))
                dest_doc[config.CONTENT_STATE] = STATE_FETCHED
                remove_unwanted(dest_doc)
                for ref in [
                    ref for group in dest_doc.get("groups", []) for ref in group.get("refs", []) if "residRef" in ref
                ]:
                    ref["location"] = ARCHIVE
                    ref["guid"] = ref["residRef"]

                set_original_creator(dest_doc)
                if doc.get(PACKAGE):
                    links = dest_doc.get(LINKED_IN_PACKAGES, [])
                    links.append({PACKAGE: doc.get(PACKAGE)})
                    dest_doc[LINKED_IN_PACKAGES] = links
                superdesk.get_resource_service(ARCHIVE).post([dest_doc])
                insert_into_versions(dest_doc.get("guid"))
                desk = doc.get("desk")
                refs = [
                    {"guid": ref.get("residRef"), "desk": desk, PACKAGE: dest_doc.get("_id")}
                    for group in dest_doc.get("groups", [])
                    for ref in group.get("refs", [])
                    if "residRef" in ref
                ]
                if refs:
                    self.create(refs)
            else:
                if doc.get(PACKAGE):
                    links = archived_doc.get(LINKED_IN_PACKAGES, [])
                    links.append({PACKAGE: doc.get(PACKAGE)})
                    superdesk.get_resource_service(ARCHIVE).patch(archived_doc.get("_id"), {LINKED_IN_PACKAGES: links})

        return [doc.get("guid") for doc in docs]
예제 #12
0
    def ingest_items_for(self, desk, no_of_stories, skip_index):
        desk_id = desk['_id']
        stage_id = desk['incoming_stage']

        bucket_size = min(100, no_of_stories)

        no_of_buckets = len(range(0, no_of_stories, bucket_size))

        for x in range(0, no_of_buckets):
            skip = x * bucket_size * skip_index
            logger.info('Page : {}, skip: {}'.format(x + 1, skip))
            cursor = get_resource_service('published').get_from_mongo(None, {})
            cursor.skip(skip)
            cursor.limit(bucket_size)
            items = list(cursor)
            logger.info('Inserting {} items'.format(len(items)))
            archive_items = []

            for item in items:
                dest_doc = dict(item)
                new_id = generate_guid(type=GUID_TAG)
                dest_doc[app.config['ID_FIELD']] = new_id
                dest_doc['guid'] = new_id
                generate_unique_id_and_name(dest_doc)

                dest_doc[app.config['VERSION']] = 1
                dest_doc[ITEM_STATE] = CONTENT_STATE.FETCHED
                user_id = desk.get('members', [{'user': None}])[0].get('user')
                dest_doc['original_creator'] = user_id
                dest_doc['version_creator'] = user_id

                from apps.tasks import send_to
                send_to(dest_doc,
                        desk_id=desk_id,
                        stage_id=stage_id,
                        user_id=user_id)
                dest_doc[app.config[
                    'VERSION']] = 1  # Above step increments the version and needs to reset
                dest_doc[FAMILY_ID] = item['_id']

                remove_unwanted(dest_doc)
                archive_items.append(dest_doc)

            get_resource_service(ARCHIVE).post(archive_items)
            for item in archive_items:
                insert_into_versions(id_=item[app.config['ID_FIELD']])
예제 #13
0
    def create(self, docs, **kwargs):
        for doc in docs:
            ingest_doc = superdesk.get_resource_service('ingest').find_one(req=None, _id=doc.get('guid'))
            if not ingest_doc:
                msg = 'Fail to found ingest item with guid: %s' % doc.get('guid')
                raise SuperdeskApiError.notFoundError(msg)

            if not is_workflow_state_transition_valid('fetch_as_from_ingest', ingest_doc[config.CONTENT_STATE]):
                raise InvalidStateTransitionError()

            archived = utcnow()
            superdesk.get_resource_service('ingest').patch(ingest_doc.get('_id'), {'archived': archived})
            doc['archived'] = archived

            archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, _id=doc.get('guid'))
            if not archived_doc:
                dest_doc = dict(ingest_doc)
                dest_doc[config.VERSION] = 1
                send_to(dest_doc, doc.get('desk'))
                dest_doc[config.CONTENT_STATE] = STATE_FETCHED
                remove_unwanted(dest_doc)
                for ref in [ref for group in dest_doc.get('groups', [])
                            for ref in group.get('refs', []) if 'residRef' in ref]:
                    ref['location'] = ARCHIVE
                    ref['guid'] = ref['residRef']

                set_original_creator(dest_doc)
                if doc.get(PACKAGE):
                    links = dest_doc.get(LINKED_IN_PACKAGES, [])
                    links.append({PACKAGE: doc.get(PACKAGE)})
                    dest_doc[LINKED_IN_PACKAGES] = links
                superdesk.get_resource_service(ARCHIVE).post([dest_doc])
                insert_into_versions(dest_doc.get('guid'))
                desk = doc.get('desk')
                refs = [{'guid': ref.get('residRef'), 'desk': desk, PACKAGE: dest_doc.get('_id')}
                        for group in dest_doc.get('groups', [])
                        for ref in group.get('refs', []) if 'residRef' in ref]
                if refs:
                    self.create(refs)
            else:
                if doc.get(PACKAGE):
                    links = archived_doc.get(LINKED_IN_PACKAGES, [])
                    links.append({PACKAGE: doc.get(PACKAGE)})
                    superdesk.get_resource_service(ARCHIVE).patch(archived_doc.get('_id'), {LINKED_IN_PACKAGES: links})

        return [doc.get('guid') for doc in docs]
예제 #14
0
    def create(self, docs, **kwargs):
        new_guids = []
        provider = get_resource_service('ingest_providers').find_one(
            source='aapmm', req=None)
        if provider and 'config' in provider and 'username' in provider[
                'config']:
            self.backend.set_credentials(provider['config']['username'],
                                         provider['config']['password'])
        for doc in docs:
            if not doc.get('desk'):
                # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError(
                    "Destination desk cannot be empty.")
            try:
                archived_doc = self.backend.find_one_raw(
                    doc['guid'], doc['guid'])
            except FileNotFoundError as ex:
                raise ProviderError.externalProviderError(ex, provider)

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc['_id'] = new_id
            generate_unique_id_and_name(dest_doc)

            if provider:
                dest_doc['ingest_provider'] = str(
                    provider[superdesk.config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc,
                    update=None,
                    desk_id=doc.get('desk'),
                    stage_id=doc.get('stage'))
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = archived_doc['_id']
            dest_doc[FAMILY_ID] = archived_doc['_id']
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc.get('_id'))

        return new_guids
예제 #15
0
    def create(self, docs, **kwargs):
        search_provider = get_resource_service('search_providers').find_one(search_provider=PROVIDER_NAME, req=None)

        if not search_provider or search_provider.get('is_closed', False):
            raise SuperdeskApiError.badRequestError('No search provider found or the search provider is closed.')

        if 'config' in search_provider:
            self.backend.set_credentials(search_provider['config'])

        new_guids = []
        for doc in docs:
            if not doc.get('desk'):  # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError("Destination desk cannot be empty.")

            try:
                archived_doc = self.backend.find_one_raw(doc['guid'], doc['guid'])
            except FileNotFoundError as ex:
                raise ProviderError.externalProviderError(ex, search_provider)

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc[config.ID_FIELD] = new_id
            generate_unique_id_and_name(dest_doc)

            if search_provider:
                dest_doc['ingest_provider'] = str(search_provider[config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc, update=None, desk_id=doc.get('desk'), stage_id=doc.get('stage'))
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = archived_doc[config.ID_FIELD]
            dest_doc[FAMILY_ID] = archived_doc[config.ID_FIELD]
            dest_doc[ITEM_OPERATION] = ITEM_FETCH
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc[config.ID_FIELD])

            get_resource_service('search_providers').system_update(search_provider[config.ID_FIELD],
                                                                   {'last_item_update': utcnow()}, search_provider)

        return new_guids
예제 #16
0
    def create(self, docs, **kwargs):
        new_guids = []
        provider = self.get_provider()
        for doc in docs:
            if not doc.get('desk'):
                # if no desk is selected then it is bad request
                raise SuperdeskApiError.badRequestError(
                    _("Destination desk cannot be empty."))
            try:
                archived_doc = self.fetch(doc['guid'])
            except FileNotFoundError as ex:
                raise ProviderError.externalProviderError(ex, provider)

            dest_doc = dict(archived_doc)
            new_id = generate_guid(type=GUID_TAG)
            new_guids.append(new_id)
            dest_doc['_id'] = new_id
            generate_unique_id_and_name(dest_doc)

            if provider:
                dest_doc['ingest_provider'] = str(
                    provider[superdesk.config.ID_FIELD])

            dest_doc[config.VERSION] = 1
            send_to(doc=dest_doc,
                    update=None,
                    desk_id=doc.get('desk'),
                    stage_id=doc.get('stage'))
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = archived_doc['_id']
            dest_doc[FAMILY_ID] = archived_doc['_id']
            dest_doc[ITEM_OPERATION] = ITEM_FETCH
            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)

            superdesk.get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(dest_doc.get('_id'))

        if new_guids:
            get_resource_service('search_providers').system_update(
                provider.get(config.ID_FIELD), {'last_item_update': utcnow()},
                provider)

        return new_guids
예제 #17
0
    def ingest_items_for(self, desk, no_of_stories, skip_index):
        desk_id = desk['_id']
        stage_id = desk['incoming_stage']

        bucket_size = min(100, no_of_stories)

        no_of_buckets = len(range(0, no_of_stories, bucket_size))

        for x in range(0, no_of_buckets):
            skip = x * bucket_size * skip_index
            logger.info('Page : {}, skip: {}'.format(x + 1, skip))
            cursor = get_resource_service('published').get_from_mongo(None, {})
            cursor.skip(skip)
            cursor.limit(bucket_size)
            items = list(cursor)
            logger.info('Inserting {} items'.format(len(items)))
            archive_items = []

            for item in items:
                dest_doc = dict(item)
                new_id = generate_guid(type=GUID_TAG)
                dest_doc[app.config['ID_FIELD']] = new_id
                dest_doc['guid'] = new_id
                generate_unique_id_and_name(dest_doc)

                dest_doc[app.config['VERSION']] = 1
                dest_doc[ITEM_STATE] = CONTENT_STATE.FETCHED
                user_id = desk.get('members', [{'user': None}])[0].get('user')
                dest_doc['original_creator'] = user_id
                dest_doc['version_creator'] = user_id

                from apps.tasks import send_to
                send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id)
                dest_doc[app.config['VERSION']] = 1  # Above step increments the version and needs to reset
                dest_doc[FAMILY_ID] = item['_id']

                remove_unwanted(dest_doc)
                archive_items.append(dest_doc)

            get_resource_service(ARCHIVE).post(archive_items)
            for item in archive_items:
                insert_into_versions(id_=item[app.config['ID_FIELD']])
예제 #18
0
    def _publish_associated_items(self, original, updates={}, publish=False):
        """Refresh and publish associated items before publishing. The publishing is done if the setting
        PUBLISH_ASSOCIATED_ITEMS was true.

        Any further updates made to basic metadata done after item was associated will be carried on and
        used when validating those items.
        """
        publish_service = None
        if config.PUBLISH_ASSOCIATED_ITEMS and publish_services.get(self.publish_type):
            publish_service = get_resource_service(publish_services[self.publish_type])

        associations = original.get(ASSOCIATIONS) or {}
        for associations_key, item in associations.items():
            if type(item) == dict and item.get(config.ID_FIELD):
                keys = [key for key in DEFAULT_SCHEMA.keys() if key not in PRESERVED_FIELDS]

                if app.settings.get('COPY_METADATA_FROM_PARENT') and item.get(ITEM_TYPE) in MEDIA_TYPES:
                    original_item = original
                    keys = FIELDS_TO_COPY_FOR_ASSOCIATED_ITEM
                else:
                    original_item = super().find_one(req=None, _id=item[config.ID_FIELD]) or {}

                update_item_data(item, original_item, keys)
                remove_unwanted(item)

                if publish_service and publish and item['type'] in MEDIA_TYPES:
                    if item.get('task', {}).get('stage', None):
                        del item['task']['stage']
                    if item['state'] not in PUBLISH_STATES:
                        get_resource_service('archive_publish').patch(id=item.pop(config.ID_FIELD), updates=item)
                    else:
                        publish_service.patch(id=item.pop(config.ID_FIELD), updates=item)
                    item['state'] = self.published_state
                    original[ASSOCIATIONS][associations_key]['state'] = self.published_state
                    original[ASSOCIATIONS][associations_key]['operation'] = self.publish_type
                    if ASSOCIATIONS not in updates:
                        updates[ASSOCIATIONS] = original[ASSOCIATIONS]
                    updates[ASSOCIATIONS][associations_key] = original[ASSOCIATIONS][associations_key]
                elif publish:
                    # Publishing an item with an association set the state and operation of the embedded association
                    original[ASSOCIATIONS][associations_key]['state'] = self.published_state
                    original[ASSOCIATIONS][associations_key]['operation'] = self.publish_type
    def ingest_items_for(self, desk, no_of_stories, skip_index):
        desk_id = desk['_id']
        stage_id = desk['incoming_stage']

        bucket_size = min(100, no_of_stories)

        no_of_buckets = len(range(0, no_of_stories, bucket_size))

        for x in range(0, no_of_buckets):
            skip = x * bucket_size * skip_index
            self.logger.info('Page : {}, skip: {}'.format(x + 1, skip))
            cursor = get_resource_service('text_archive').get_from_mongo(None, {})
            cursor.skip(skip)
            cursor.limit(bucket_size)
            items = list(cursor)
            self.logger.info('Inserting {} items'.format(len(items)))
            archive_items = []

            for item in items:
                dest_doc = dict(item)
                new_id = generate_guid(type=GUID_TAG)
                dest_doc['_id'] = new_id
                dest_doc['guid'] = new_id
                generate_unique_id_and_name(dest_doc)

                dest_doc[app.config['VERSION']] = 1
                dest_doc['state'] = 'fetched'
                user_id = desk.get('members', [{'user': None}])[0].get('user')
                dest_doc['original_creator'] = user_id
                dest_doc['version_creator'] = user_id
                send_to(dest_doc, desk_id=desk_id, stage_id=stage_id, user_id=user_id)
                dest_doc[FAMILY_ID] = item['_id']

                remove_unwanted(dest_doc)
                archive_items.append(dest_doc)

            get_resource_service(ARCHIVE).post(archive_items)
            for item in archive_items:
                insert_into_versions(id_=item['_id'])
예제 #20
0
    def _publish_associated_items(self, original, updates=None):
        """If there any updates to associated item and if setting:PUBLISH_ASSOCIATED_ITEMS is true
        then publish the associated item
        """

        if updates is None:
            updates = {}

        if not publish_services.get(self.publish_type):
            # publish type not supported
            return

        publish_service = get_resource_service(
            publish_services.get(self.publish_type))

        if not updates.get(ASSOCIATIONS) and not original.get(ASSOCIATIONS):
            # there's nothing to update
            return

        associations = original.get(ASSOCIATIONS) or {}

        if updates and updates.get(ASSOCIATIONS):
            associations.update(updates[ASSOCIATIONS])

        for associations_key, associated_item in associations.items():
            if associated_item is None:
                continue

            if type(associated_item) == dict and associated_item.get(
                    config.ID_FIELD):

                if not config.PUBLISH_ASSOCIATED_ITEMS or not publish_service:
                    # Not allowed to publish
                    original[ASSOCIATIONS][associations_key][
                        'state'] = self.published_state
                    original[ASSOCIATIONS][associations_key][
                        'operation'] = self.publish_type
                    continue

                # if item is not fetchable, only mark it as published
                if not associated_item.get('_fetchable', True):
                    associated_item['state'] = self.published_state
                    associated_item['operation'] = self.publish_type
                    updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
                    updates[ASSOCIATIONS][associations_key] = associated_item
                    continue

                if associated_item.get('state') == CONTENT_STATE.UNPUBLISHED:
                    # get the original associated item from archive
                    orig_associated_item = get_resource_service('archive') \
                        .find_one(req=None, _id=associated_item[config.ID_FIELD])

                    orig_associated_item['state'] = self.published_state
                    orig_associated_item['operation'] = self.publish_type

                    get_resource_service('archive_publish').patch(
                        id=orig_associated_item.pop(config.ID_FIELD),
                        updates=orig_associated_item)
                    continue

                if associated_item.get('state') not in PUBLISH_STATES:
                    # This associated item has not been published before
                    associated_item.get('task', {}).pop('stage', None)
                    remove_unwanted(associated_item)

                    # get the original associated item from archive
                    original_associated_item = get_resource_service('archive').\
                        find_one(req=None, _id=associated_item[config.ID_FIELD])

                    # check if the original associated item exists in archive
                    if not original_associated_item:
                        raise SuperdeskApiError.badRequestError(
                            _('Associated item "{}" does not exist in the system'
                              .format(associations_key)))

                    if original_associated_item.get('state') in PUBLISH_STATES:
                        # item was published already
                        original[ASSOCIATIONS][associations_key].update({
                            'state':
                            original_associated_item['state'],
                            'operation':
                            original_associated_item.get(
                                'operation', self.publish_type),
                        })
                        continue

                    # update _updated, otherwise it's stored as string.
                    # fixes SDESK-5043
                    associated_item['_updated'] = utcnow()

                    get_resource_service('archive_publish').patch(
                        id=associated_item.pop(config.ID_FIELD),
                        updates=associated_item)
                    associated_item['state'] = self.published_state
                    associated_item['operation'] = self.publish_type
                    updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
                    updates[ASSOCIATIONS][associations_key] = associated_item
                elif associated_item.get('state') != self.published_state:
                    # Check if there are updates to associated item
                    association_updates = updates.get(ASSOCIATIONS,
                                                      {}).get(associations_key)

                    if not association_updates:
                        # there is no update for this item
                        associated_item.get('task', {}).pop('stage', None)
                        remove_unwanted(associated_item)
                        publish_service.patch(id=associated_item.pop(
                            config.ID_FIELD),
                                              updates=associated_item)
                        continue

                    if association_updates.get('state') not in PUBLISH_STATES:
                        # There's an update to the published associated item
                        remove_unwanted(association_updates)
                        publish_service.patch(id=association_updates.pop(
                            config.ID_FIELD),
                                              updates=association_updates)

        self._refresh_associated_items(original)
예제 #21
0
    def fetch(self, docs, id=None, **kwargs):
        id_of_fetched_items = []

        for doc in docs:
            id_of_item_to_be_fetched = doc.get(
                config.ID_FIELD) if id is None else id

            desk_id = doc.get('desk')
            stage_id = doc.get('stage')

            ingest_service = get_resource_service('ingest')
            ingest_doc = ingest_service.find_one(req=None,
                                                 _id=id_of_item_to_be_fetched)

            if not ingest_doc:
                raise SuperdeskApiError.notFoundError(
                    _('Fail to found ingest item with _id: {id}').format(
                        id=id_of_item_to_be_fetched))

            if not is_workflow_state_transition_valid('fetch_from_ingest',
                                                      ingest_doc[ITEM_STATE]):
                raise InvalidStateTransitionError()

            if doc.get('macro'):  # there is a macro so transform it
                ingest_doc = get_resource_service('macros').execute_macro(
                    ingest_doc, doc.get('macro'))

            archived = utcnow()
            ingest_service.patch(id_of_item_to_be_fetched,
                                 {'archived': archived})

            dest_doc = dict(ingest_doc)

            if doc.get('target'):
                dest_doc.update(doc.get('target'))

            new_id = generate_guid(type=GUID_TAG)
            id_of_fetched_items.append(new_id)
            dest_doc[config.ID_FIELD] = new_id
            dest_doc[GUID_FIELD] = new_id
            generate_unique_id_and_name(dest_doc)

            dest_doc[config.VERSION] = 1
            dest_doc['versioncreated'] = archived
            send_to(doc=dest_doc, desk_id=desk_id, stage_id=stage_id)
            dest_doc[ITEM_STATE] = doc.get(ITEM_STATE, CONTENT_STATE.FETCHED)
            dest_doc[INGEST_ID] = dest_doc[FAMILY_ID] = ingest_doc[
                config.ID_FIELD]
            dest_doc[ITEM_OPERATION] = ITEM_FETCH

            remove_unwanted(dest_doc)
            set_original_creator(dest_doc)
            self.__fetch_items_in_package(
                dest_doc, desk_id, stage_id,
                doc.get(ITEM_STATE, CONTENT_STATE.FETCHED))

            self.__fetch_associated_items(
                dest_doc, desk_id, stage_id,
                doc.get(ITEM_STATE, CONTENT_STATE.FETCHED))

            desk = get_resource_service('desks').find_one(req=None,
                                                          _id=desk_id)
            if desk and desk.get('default_content_profile'):
                dest_doc['profile'] = desk['default_content_profile']

            if dest_doc.get('type', 'text') in MEDIA_TYPES:
                dest_doc['profile'] = None

            get_resource_service(ARCHIVE).post([dest_doc])
            insert_into_versions(doc=dest_doc)
            build_custom_hateoas(custom_hateoas, dest_doc)
            superdesk.item_fetched.send(self,
                                        item=dest_doc,
                                        ingest_item=ingest_doc)
            doc.update(dest_doc)

        if kwargs.get('notify', True):
            ingest_doc.update({'task': dest_doc.get('task')})
            push_item_move_notification(ingest_doc, doc, 'item:fetch')

        return id_of_fetched_items
예제 #22
0
    def _publish_associated_items(self, original, updates={}):
        """If there any updates to associated item and if setting:PUBLISH_ASSOCIATED_ITEMS is true
        then publish the associated item
        """
        if not publish_services.get(self.publish_type):
            # publish type not supported
            return

        publish_service = get_resource_service(
            publish_services.get(self.publish_type))

        if not updates.get(ASSOCIATIONS) and not original.get(ASSOCIATIONS):
            # there's nothing to update
            return

        associations = original.get(ASSOCIATIONS) or {}
        for associations_key, associated_item in associations.items():
            if type(associated_item) == dict and associated_item.get(
                    config.ID_FIELD):

                if not config.PUBLISH_ASSOCIATED_ITEMS or not publish_service:
                    # Not allowed to publish
                    original[ASSOCIATIONS][associations_key][
                        'state'] = self.published_state
                    original[ASSOCIATIONS][associations_key][
                        'operation'] = self.publish_type
                    continue

                if associated_item.get('state') not in PUBLISH_STATES:
                    # This associated item has not been published before
                    associated_item.get('task', {}).pop('stage', None)
                    remove_unwanted(associated_item)

                    # get the original associated item from archive
                    original_associated_item = get_resource_service('archive').\
                        find_one(req=None, _id=associated_item[config.ID_FIELD])

                    # check if the original associated item exists in archive
                    if not original_associated_item:
                        raise SuperdeskApiError.badRequestError(
                            _('Associated item "{}" does not exist in the system'
                              .format(associations_key)))

                    get_resource_service('archive_publish').patch(
                        id=associated_item.pop(config.ID_FIELD),
                        updates=associated_item)
                    associated_item['state'] = self.published_state
                    associated_item['operation'] = self.publish_type
                    updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
                    updates[ASSOCIATIONS][associations_key] = associated_item
                else:
                    # Check if there are updates to associated item
                    association_updates = updates.get(ASSOCIATIONS,
                                                      {}).get(associations_key)

                    if not association_updates:
                        # there is no update for this item but still patching the associated item
                        associated_item.get('task', {}).pop('stage', None)
                        remove_unwanted(associated_item)
                        publish_service.patch(id=associated_item.pop(
                            config.ID_FIELD),
                                              updates=associated_item)
                        continue

                    if association_updates.get('state') not in PUBLISH_STATES:
                        # There's an update to the published associated item
                        remove_unwanted(association_updates)
                        publish_service.patch(id=association_updates.pop(
                            config.ID_FIELD),
                                              updates=association_updates)

        self._refresh_associated_items(original)
예제 #23
0
    def _publish_associated_items(self, original, updates=None):
        """If there any updates to associated item and if setting:PUBLISH_ASSOCIATED_ITEMS is true
        then publish the associated item
        """

        if updates is None:
            updates = {}

        if not publish_services.get(self.publish_type):
            # publish type not supported
            return

        publish_service = get_resource_service(publish_services.get(self.publish_type))

        if not updates.get(ASSOCIATIONS) and not original.get(ASSOCIATIONS):
            # there's nothing to update
            return

        associations = original.get(ASSOCIATIONS) or {}

        if updates and updates.get(ASSOCIATIONS):
            associations.update(updates[ASSOCIATIONS])

        archive_service = get_resource_service("archive")

        for associations_key, associated_item in associations.items():
            if associated_item is None:
                continue
            if type(associated_item) == dict and associated_item.get(config.ID_FIELD):
                if not config.PUBLISH_ASSOCIATED_ITEMS or not publish_service:
                    if original.get(ASSOCIATIONS, {}).get(associations_key):
                        # Not allowed to publish
                        original[ASSOCIATIONS][associations_key]["state"] = self.published_state
                        original[ASSOCIATIONS][associations_key]["operation"] = self.publish_type
                    continue

                # if item is not fetchable, only mark it as published
                if not associated_item.get("_fetchable", True):
                    associated_item["state"] = self.published_state
                    associated_item["operation"] = self.publish_type
                    updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
                    updates[ASSOCIATIONS][associations_key] = associated_item
                    continue

                if associated_item.get("state") == CONTENT_STATE.UNPUBLISHED:
                    # get the original associated item from archive
                    orig_associated_item = archive_service.find_one(req=None, _id=associated_item[config.ID_FIELD])

                    orig_associated_item["state"] = updates.get("state", self.published_state)
                    orig_associated_item["operation"] = self.publish_type

                    # if main item is scheduled we must also schedule associations
                    self._inherit_publish_schedule(original, updates, orig_associated_item)

                    get_resource_service("archive_publish").patch(
                        id=orig_associated_item.pop(config.ID_FIELD), updates=orig_associated_item
                    )
                    continue

                if associated_item.get("state") not in PUBLISH_STATES:
                    # This associated item has not been published before
                    remove_unwanted(associated_item)

                    # get the original associated item from archive
                    orig_associated_item = archive_service.find_one(req=None, _id=associated_item[config.ID_FIELD])

                    # check if the original associated item exists in archive
                    if not orig_associated_item:
                        raise SuperdeskApiError.badRequestError(
                            _('Associated item "{}" does not exist in the system'.format(associations_key))
                        )

                    if orig_associated_item.get("state") in PUBLISH_STATES:
                        # item was published already
                        original[ASSOCIATIONS][associations_key].update(
                            {
                                "state": orig_associated_item["state"],
                                "operation": orig_associated_item.get("operation", self.publish_type),
                            }
                        )
                        continue

                    # if the original associated item stage is present, it should be updated in the association item.
                    if orig_associated_item.get("task", {}).get("stage") and associated_item.get("task"):
                        associated_item["task"].update({"stage": orig_associated_item.get("task", {}).get("stage")})

                    # update _updated, otherwise it's stored as string.
                    # fixes SDESK-5043
                    associated_item["_updated"] = utcnow()

                    # if main item is scheduled we must also schedule associations
                    self._inherit_publish_schedule(original, updates, associated_item)

                    get_resource_service("archive_publish").patch(
                        id=associated_item.pop(config.ID_FIELD), updates=associated_item
                    )
                    associated_item["state"] = updates.get("state", self.published_state)
                    associated_item["operation"] = self.publish_type
                    updates[ASSOCIATIONS] = updates.get(ASSOCIATIONS, {})
                    updates[ASSOCIATIONS][associations_key] = associated_item
                elif associated_item.get("state") != self.published_state:
                    # Check if there are updates to associated item
                    association_updates = updates.get(ASSOCIATIONS, {}).get(associations_key)

                    # if main item is scheduled we must also schedule associations
                    self._inherit_publish_schedule(original, updates, associated_item)

                    if not association_updates:
                        # there is no update for this item
                        associated_item.get("task", {}).pop("stage", None)
                        remove_unwanted(associated_item)
                        publish_service.patch(id=associated_item.pop(config.ID_FIELD), updates=associated_item)
                        continue

                    if association_updates.get("state") not in PUBLISH_STATES:
                        # There's an update to the published associated item
                        remove_unwanted(association_updates)
                        publish_service.patch(id=association_updates.pop(config.ID_FIELD), updates=association_updates)

            # When there is an associated item which is published, Inserts the latest version of that associated item into archive_versions.
            insert_into_versions(doc=associated_item)
        self._refresh_associated_items(original)
    def create(self, docs, **kwargs):
        new_guids = []
        for doc in docs:
            ingest_doc = superdesk.get_resource_service('ingest').find_one(req=None, _id=doc.get('guid'))
            if not ingest_doc:
                # see if it is in archive, if it is duplicate it
                archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, _id=doc.get('guid'))
                if archived_doc:
                    send_to(archived_doc, doc.get('desk'))
                    new_guid = superdesk.get_resource_service('archive').duplicate_content(archived_doc)
                    new_guids.append(new_guid)
                else:
                    msg = 'Fail to found ingest item with guid: %s' % doc.get('guid')
                    raise SuperdeskApiError.notFoundError(msg)
            else:
                # We are fetching from ingest
                if not is_workflow_state_transition_valid('fetch_as_from_ingest', ingest_doc[config.CONTENT_STATE]):
                    raise InvalidStateTransitionError()

                archived = utcnow()
                superdesk.get_resource_service('ingest').patch(ingest_doc.get('_id'), {'archived': archived})
                doc['archived'] = archived

                dest_doc = dict(ingest_doc)
                new_id = generate_guid(type=GUID_TAG)
                new_guids.append(new_id)
                dest_doc['_id'] = new_id
                dest_doc['guid'] = new_id
                generate_unique_id_and_name(dest_doc)

                dest_doc[config.VERSION] = 1
                send_to(dest_doc, doc.get('desk'))
                dest_doc[config.CONTENT_STATE] = STATE_FETCHED
                dest_doc[INGEST_ID] = ingest_doc['_id']
                dest_doc[FAMILY_ID] = ingest_doc['_id']
                remove_unwanted(dest_doc)
                for ref in [ref for group in dest_doc.get('groups', [])
                            for ref in group.get('refs', []) if 'residRef' in ref]:
                    ref['location'] = ARCHIVE
                    ref['guid'] = ref['residRef']

                set_original_creator(dest_doc)
                if doc.get(PACKAGE):
                    links = dest_doc.get(LINKED_IN_PACKAGES, [])
                    links.append({PACKAGE: doc.get(PACKAGE)})
                    dest_doc[LINKED_IN_PACKAGES] = links

                desk = doc.get('desk')
                refs = [{'guid': ref.get('residRef'), 'desk': desk, PACKAGE: dest_doc.get('_id')}
                        for group in dest_doc.get('groups', [])
                        for ref in group.get('refs', []) if 'residRef' in ref]
                if refs:
                    new_ref_guids = self.create(refs)
                    count = 0
                    for ref in [ref for group in dest_doc.get('groups', [])
                                for ref in group.get('refs', []) if 'residRef' in ref]:
                        ref['residRef'] = ref['guid'] = new_ref_guids[count]
                        count += 1

                superdesk.get_resource_service(ARCHIVE).post([dest_doc])
                insert_into_versions(dest_doc.get('guid'))

                push_notification('item:fetch', item=str(ingest_doc.get('_id')))

        return new_guids