def archive_rendition(self, task_id, guid, name, href, trigger_events):
    try:
        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))
        import_rendition(guid, name, href, False, trigger_events)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)
def archive_media(self, task_id, guid, href):
    try:
        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))
        import_rendition(guid, 'baseImage', href, True)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)
Example #3
0
def archive_rendition(self, task_id, guid, name, href, trigger_events):
    try:
        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))
        import_rendition(guid, name, href, False, trigger_events)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)
def archive_media(self, task_id, guid, href):
    try:
        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))
        import_rendition(guid, 'baseImage', href, True)
        update_status(*finish_subtask_from_progress(task_id))
    except Exception:
        raise self.retry(countdown=2)
def archive_item(self, guid, provider_id, user, trigger_events, task_id=None):
    try:
        # For CELERY_ALWAYS_EAGER=True the current request context is
        # empty but already initialized one is on request_stack
        if app.config["CELERY_ALWAYS_EAGER"]:
            self.request_stack.pop()

        crt_task_id = self.request.id
        if not task_id:
            task_id = crt_task_id

        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))

        provider = superdesk.apps["ingest_providers"].find_one(req=None, _id=provider_id)
        if provider is None:
            message = "For ingest with guid= %s, failed to retrieve provider with _id=%s" % (guid, provider_id)
            raise_fail(task_id, message)
        service_provider = superdesk.io.providers[provider.get("type")]
        service_provider.provider = provider

        item = None
        old_item = False
        try:
            items = service_provider.get_items(guid)
        except LookupError:
            ingest_doc = superdesk.apps["ingest"].find_one(req=None, _id=guid)
            if not ingest_doc:
                message = "Not found the ingest with guid: %s for provider %s" % (guid, provider.get("type"))
                raise_fail(task_id, message)
            else:
                old_item = True
                ingest_doc.pop("_id")
                items = [ingest_doc]
        except Exception:
            raise self.retry(countdown=2)

        for item_it in items:
            if "guid" in item_it and item_it["guid"] == guid:
                item = item_it
                break

        if item is None:
            message = "Returned ingest but not found the ingest with guid: %s for provider %s" % (
                guid,
                provider.get("type"),
            )
            raise_fail(task_id, message)

        if not old_item:
            item["created"] = item["firstcreated"] = utc.localize(item["firstcreated"])
            item["updated"] = item["versioncreated"] = utc.localize(item["versioncreated"])
        superdesk.apps["archive"].update(guid, item, trigger_events=trigger_events)

        tasks = []
        for group in item.get("groups", []):
            for ref in group.get("refs", []):
                if "residRef" in ref:
                    doc = {
                        "guid": ref.get("residRef"),
                        "ingest_provider": provider_id,
                        "user": user,
                        "task_id": crt_task_id,
                    }

                    archived_doc = superdesk.apps["archive"].find_one(req=None, guid=doc.get("guid"))
                    # check if task already started
                    if not archived_doc:
                        doc.setdefault("_id", doc.get("guid"))
                        superdesk.apps["archive"].create([doc], trigger_events=trigger_events)
                    elif archived_doc.get("task_id") == crt_task_id:
                        # it is a retry so continue
                        archived_doc.update(doc)
                        superdesk.apps["archive"].update(
                            archived_doc.get("_id"), archived_doc, trigger_events=trigger_events
                        )
                    else:
                        # there is a cyclic dependency, skip it
                        continue

                    ingest_set_archived(doc.get("guid"))
                    tasks.append(archive_item.s(ref["residRef"], provider.get("_id"), user, trigger_events, task_id))

        for rendition in item.get("renditions", {}).values():
            href = service_provider.prepare_href(rendition["href"])
            if rendition["rendition"] == "baseImage":
                tasks.append(archive_media.s(task_id, guid, href, trigger_events))
            else:
                tasks.append(archive_rendition.s(task_id, guid, rendition["rendition"], href, trigger_events))

        update_status(*finish_subtask_from_progress(task_id))
        if tasks:
            chord((task for task in tasks), update_item.s(crt_task_id == task_id, task_id, guid)).delay()
        elif task_id == crt_task_id:
            update_status(*finish_task_for_progress(task_id))
    except Exception:
        logger.error(traceback.format_exc())
Example #6
0
def archive_item(
    self,
    guid,
    provider_id,
    user,
    trigger_events,
    task_id=None,
):
    try:
        # For CELERY_ALWAYS_EAGER=True the current request context is
        # empty but already initialized one is on request_stack
        if app.config['CELERY_ALWAYS_EAGER']:
            self.request_stack.pop()

        crt_task_id = self.request.id
        if not task_id:
            task_id = crt_task_id

        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))

        provider = superdesk.apps['ingest_providers'].find_one(req=None,
                                                               _id=provider_id)
        if provider is None:
            message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (
                guid, provider_id)
            raise_fail(task_id, message)
        service_provider = superdesk.io.providers[provider.get('type')]
        service_provider.provider = provider

        item = None
        old_item = False
        try:
            items = service_provider.get_items(guid)
        except LookupError:
            ingest_doc = superdesk.apps['ingest'].find_one(req=None, _id=guid)
            if not ingest_doc:
                message = 'Not found the ingest with guid: %s for provider %s' % (
                    guid, provider.get('type'))
                raise_fail(task_id, message)
            else:
                old_item = True
                ingest_doc.pop('_id')
                items = [ingest_doc]
        except Exception:
            raise self.retry(countdown=2)

        for item_it in items:
            if 'guid' in item_it and item_it['guid'] == guid:
                item = item_it
                break

        if item is None:
            message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
                      % (guid, provider.get('type'))
            raise_fail(task_id, message)

        if not old_item:
            item['created'] = item['firstcreated'] = utc.localize(
                item['firstcreated'])
            item['updated'] = item['versioncreated'] = utc.localize(
                item['versioncreated'])
        superdesk.apps['archive'].update(guid,
                                         item,
                                         trigger_events=trigger_events)

        tasks = []
        for group in item.get('groups', []):
            for ref in group.get('refs', []):
                if 'residRef' in ref:
                    doc = {
                        'guid': ref.get('residRef'),
                        'ingest_provider': provider_id,
                        'user': user,
                        'task_id': crt_task_id
                    }

                    archived_doc = superdesk.apps['archive'].find_one(
                        req=None, guid=doc.get('guid'))
                    # check if task already started
                    if not archived_doc:
                        doc.setdefault('_id', doc.get('guid'))
                        superdesk.apps['archive'].create(
                            [doc], trigger_events=trigger_events)
                    elif archived_doc.get('task_id') == crt_task_id:
                        # it is a retry so continue
                        archived_doc.update(doc)
                        superdesk.apps['archive'].update(
                            archived_doc.get('_id'),
                            archived_doc,
                            trigger_events=trigger_events)
                    else:
                        # there is a cyclic dependency, skip it
                        continue

                    ingest_set_archived(doc.get('guid'))
                    tasks.append(
                        archive_item.s(ref['residRef'], provider.get('_id'),
                                       user, trigger_events, task_id))

        for rendition in item.get('renditions', {}).values():
            href = service_provider.prepare_href(rendition['href'])
            if rendition['rendition'] == 'baseImage':
                tasks.append(
                    archive_media.s(task_id, guid, href, trigger_events))
            else:
                tasks.append(
                    archive_rendition.s(task_id, guid, rendition['rendition'],
                                        href, trigger_events))

        update_status(*finish_subtask_from_progress(task_id))
        if tasks:
            chord((task for task in tasks),
                  update_item.s(crt_task_id == task_id, task_id,
                                guid)).delay()
        elif task_id == crt_task_id:
            update_status(*finish_task_for_progress(task_id))
    except Exception:
        logger.error(traceback.format_exc())
def archive_item(self, guid, provider_id, user, task_id=None):
    try:
        # For CELERY_ALWAYS_EAGER=True the current request context is
        # empty but already initialized one is on request_stack
        if app.config['CELERY_ALWAYS_EAGER']:
            self.request_stack.pop()

        crt_task_id = self.request.id
        if not task_id:
            task_id = crt_task_id

        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))

        provider = superdesk.get_resource_service('ingest_providers').find_one(req=None, _id=provider_id)
        if provider is None:
            message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (guid, provider_id)
            raise_fail(task_id, message)
        service_provider = superdesk.io.providers[provider.get('type')]
        service_provider.provider = provider

        item = None
        old_item = False
        try:
            items = service_provider.get_items(guid)
        except LookupError:
            ingest_doc = superdesk.get_resource_service('ingest').find_one(req=None, _id=guid)
            if not ingest_doc:
                message = 'Not found the ingest with guid: %s for provider %s' % (guid, provider.get('type'))
                raise_fail(task_id, message)
            else:
                old_item = True
                ingest_doc.pop('_id')
                items = [ingest_doc]
        except Exception:
            raise self.retry(countdown=2)

        for item_it in items:
            if 'guid' in item_it and item_it['guid'] == guid:
                item = item_it
                break

        if item is None:
            message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
                      % (guid, provider.get('type'))
            raise_fail(task_id, message)

        if not old_item:
            item['created'] = item['firstcreated'] = utc.localize(item['firstcreated'])
            item['updated'] = item['versioncreated'] = utc.localize(item['versioncreated'])

        '''
        Necessary because flask.g.user is None while fetching packages the for grouped items or
        while patching in archive collection. Without this version_creator is set None which doesn't make sense.
        '''
        flask.g.user = user
        remove_unwanted(item)
        superdesk.get_resource_service(ARCHIVE).patch(guid, item)

        tasks = []
        for group in item.get('groups', []):
            for ref in group.get('refs', []):
                if 'residRef' in ref:
                    resid_ref = ref.get('residRef')
                    doc = {'guid': resid_ref, 'ingest_provider': provider_id, 'task_id': crt_task_id}

                    archived_doc = superdesk.get_resource_service(ARCHIVE).find_one(req=None, guid=doc.get('guid'))
                    # check if task already started
                    if not archived_doc:
                        doc.setdefault('_id', doc.get('guid'))
                        superdesk.get_resource_service(ARCHIVE).post([doc])
                    elif archived_doc.get('task_id') == crt_task_id:
                        # it is a retry so continue
                        archived_doc.update(doc)
                        remove_unwanted(archived_doc)
                        superdesk.get_resource_service(ARCHIVE).patch(archived_doc.get('_id'), archived_doc)
                    else:
                        # there is a cyclic dependency, skip it
                        continue

                    mark_ingest_as_archived(doc.get('guid'))
                    tasks.append(archive_item.s(resid_ref, provider.get('_id'), user, task_id))

        for rendition in item.get('renditions', {}).values():
            href = service_provider.prepare_href(rendition['href'])
            if rendition['rendition'] == 'baseImage':
                tasks.append(archive_media.s(task_id, guid, href))
            else:
                tasks.append(archive_rendition.s(task_id, guid, rendition['rendition'], href))

        update_status(*finish_subtask_from_progress(task_id))
        if tasks:
            chord((task for task in tasks), update_item.s(crt_task_id == task_id, task_id, guid)).delay()
        else:
            insert_into_versions(guid, task_id)
            if task_id == crt_task_id:
                update_status(*finish_task_for_progress(task_id))
    except Exception:
        logger.error(traceback.format_exc())
def archive_item(self, guid, provider_id, user, trigger_events, task_id=None, ):
    try:
        # For CELERY_ALWAYS_EAGER=True the current request context is
        # empty but already initialized one is on request_stack
        if app.config['CELERY_ALWAYS_EAGER']:
            self.request_stack.pop()

        crt_task_id = self.request.id
        if not task_id:
            task_id = crt_task_id

        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))

        provider = superdesk.apps['ingest_providers'].find_one(req=None, _id=provider_id)
        if provider is None:
            message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (guid, provider_id)
            raise_fail(task_id, message)
        service_provider = providers[provider.get('type')]
        service_provider.provider = provider

        item = None
        old_item = False
        try:
            items = service_provider.get_items(guid)
        except LookupError:
            ingest_doc = superdesk.apps['ingest'].find_one(req=None, _id=guid)
            if not ingest_doc:
                message = 'Not found the ingest with guid: %s for provider %s' % (guid, provider.get('type'))
                raise_fail(task_id, message)
            else:
                old_item = True
                ingest_doc.pop('_id')
                items = [ingest_doc]
        except Exception:
            raise self.retry(countdown=2)

        for item_it in items:
            if 'guid' in item_it and item_it['guid'] == guid:
                item = item_it
                break

        if item is None:
            message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
                      % (guid, provider.get('type'))
            raise_fail(task_id, message)

        if not old_item:
            item['created'] = item['firstcreated'] = utc.localize(item['firstcreated'])
            item['updated'] = item['versioncreated'] = utc.localize(item['versioncreated'])
        superdesk.apps['archive'].update(guid, item, trigger_events=trigger_events)

        tasks = []
        for group in item.get('groups', []):
            for ref in group.get('refs', []):
                if 'residRef' in ref:
                    doc = {'guid': ref.get('residRef'), 'ingest_provider': provider_id,
                           'user': user, 'task_id': crt_task_id}

                    archived_doc = superdesk.apps['archive'].find_one(req=None, guid=doc.get('guid'))
                    # check if task already started
                    if not archived_doc:
                        doc.setdefault('_id', doc.get('guid'))
                        superdesk.apps['archive'].create([doc], trigger_events=trigger_events)
                    elif archived_doc.get('task_id') == crt_task_id:
                        # it is a retry so continue
                        archived_doc.update(doc)
                        superdesk.apps['archive'].update(archived_doc.get('_id'), archived_doc,
                                                         trigger_events=trigger_events)
                    else:
                        # there is a cyclic dependency, skip it
                        continue

                    ingest_set_archived(doc.get('guid'))
                    tasks.append(archive_item.s(ref['residRef'], provider.get('_id'), user, trigger_events, task_id))

        for rendition in item.get('renditions', {}).values():
            href = service_provider.prepare_href(rendition['href'])
            if rendition['rendition'] == 'baseImage':
                tasks.append(archive_media.s(task_id, guid, href, trigger_events))
            else:
                tasks.append(archive_rendition.s(task_id, guid, rendition['rendition'], href, trigger_events))

        update_status(*finish_subtask_from_progress(task_id))
        if tasks:
            chord((task for task in tasks), update_item.s(crt_task_id == task_id, task_id, guid)).delay()
        elif task_id == crt_task_id:
            update_status(*finish_task_for_progress(task_id))
    except Exception:
        logger.error(traceback.format_exc())
Example #9
0
def archive_item(self, guid, provider_id, user, task_id=None):
    try:
        # For CELERY_ALWAYS_EAGER=True the current request context is
        # empty but already initialized one is on request_stack
        if app.config['CELERY_ALWAYS_EAGER']:
            self.request_stack.pop()

        crt_task_id = self.request.id
        if not task_id:
            task_id = crt_task_id

        if not self.request.retries:
            update_status(*add_subtask_to_progress(task_id))

        provider = superdesk.get_resource_service('ingest_providers').find_one(
            req=None, _id=provider_id)
        if provider is None:
            message = 'For ingest with guid= %s, failed to retrieve provider with _id=%s' % (
                guid, provider_id)
            raise_fail(task_id, message)
        service_provider = superdesk.io.providers[provider.get('type')]
        service_provider.provider = provider

        item = None
        old_item = False
        try:
            items = service_provider.get_items(guid)
        except LookupError:
            ingest_doc = superdesk.get_resource_service('ingest').find_one(
                req=None, _id=guid)
            if not ingest_doc:
                message = 'Not found the ingest with guid: %s for provider %s' % (
                    guid, provider.get('type'))
                raise_fail(task_id, message)
            else:
                old_item = True
                ingest_doc.pop('_id')
                items = [ingest_doc]
        except Exception:
            raise self.retry(countdown=2)

        for item_it in items:
            if 'guid' in item_it and item_it['guid'] == guid:
                item = item_it
                break

        if item is None:
            message = 'Returned ingest but not found the ingest with guid: %s for provider %s' \
                      % (guid, provider.get('type'))
            raise_fail(task_id, message)

        if not old_item:
            item['created'] = item['firstcreated'] = utc.localize(
                item['firstcreated'])
            item['updated'] = item['versioncreated'] = utc.localize(
                item['versioncreated'])
        '''
        Necessary because flask.g.user is None while fetching packages the for grouped items or
        while patching in archive collection. Without this version_creator is set None which doesn't make sense.
        '''
        flask.g.user = user
        remove_unwanted(item)
        superdesk.get_resource_service(ARCHIVE).patch(guid, item)

        tasks = []
        for group in item.get('groups', []):
            for ref in group.get('refs', []):
                if 'residRef' in ref:
                    resid_ref = ref.get('residRef')
                    doc = {
                        'guid': resid_ref,
                        'ingest_provider': provider_id,
                        'task_id': crt_task_id
                    }

                    archived_doc = superdesk.get_resource_service(
                        ARCHIVE).find_one(req=None, guid=doc.get('guid'))
                    # check if task already started
                    if not archived_doc:
                        doc.setdefault('_id', doc.get('guid'))
                        superdesk.get_resource_service(ARCHIVE).post([doc])
                    elif archived_doc.get('task_id') == crt_task_id:
                        # it is a retry so continue
                        archived_doc.update(doc)
                        remove_unwanted(archived_doc)
                        superdesk.get_resource_service(ARCHIVE).patch(
                            archived_doc.get('_id'), archived_doc)
                    else:
                        # there is a cyclic dependency, skip it
                        continue

                    mark_ingest_as_archived(doc.get('guid'))
                    tasks.append(
                        archive_item.s(resid_ref, provider.get('_id'), user,
                                       task_id))

        for rendition in item.get('renditions', {}).values():
            href = service_provider.prepare_href(rendition['href'])
            if rendition['rendition'] == 'baseImage':
                tasks.append(archive_media.s(task_id, guid, href))
            else:
                tasks.append(
                    archive_rendition.s(task_id, guid, rendition['rendition'],
                                        href))

        update_status(*finish_subtask_from_progress(task_id))
        if tasks:
            chord((task for task in tasks),
                  update_item.s(crt_task_id == task_id, task_id,
                                guid)).delay()
        else:
            insert_into_versions(guid, task_id)
            if task_id == crt_task_id:
                update_status(*finish_task_for_progress(task_id))
    except Exception:
        logger.error(traceback.format_exc())