예제 #1
0
    def test_the_second_update_fails_if_already_running(self):
        provider = {
            "_id": "abc",
            "name": "test provider",
            "update_schedule": {
                "minutes": 1
            }
        }
        removed = mark_task_as_not_running(provider["name"], provider["_id"])
        self.assertFalse(removed)
        failed_to_mark_as_running = is_task_running(provider["name"],
                                                    provider["_id"],
                                                    {"minutes": 1})
        self.assertFalse(failed_to_mark_as_running,
                         "Failed to mark ingest update as running")

        failed_to_mark_as_running = is_task_running(provider["name"],
                                                    provider["_id"],
                                                    {"minutes": 1})
        self.assertTrue(
            failed_to_mark_as_running,
            "Ingest update marked as running, possible race condition")

        removed = mark_task_as_not_running(provider["name"], provider["_id"])
        self.assertTrue(removed,
                        "Failed to mark ingest update as not running.")
    def test_the_second_update_fails_if_already_running(self):
        provider = {
            '_id': 'abc',
            'name': 'test provider',
            'update_schedule': {
                'minutes': 1
            }
        }
        removed = mark_task_as_not_running(provider['name'], provider['_id'])
        self.assertFalse(removed)
        failed_to_mark_as_running = is_task_running(provider['name'],
                                                    provider['_id'],
                                                    {'minutes': 1})
        self.assertFalse(failed_to_mark_as_running,
                         'Failed to mark ingest update as running')

        failed_to_mark_as_running = is_task_running(provider['name'],
                                                    provider['_id'],
                                                    {'minutes': 1})
        self.assertTrue(
            failed_to_mark_as_running,
            'Ingest update marked as running, possible race condition')

        removed = mark_task_as_not_running(provider['name'], provider['_id'])
        self.assertTrue(removed,
                        'Failed to mark ingest update as not running.')
    def test_the_second_update_fails_if_already_running(self):
        provider = {'_id': 'abc', 'name': 'test provider', 'update_schedule': {'minutes': 1}}
        removed = mark_task_as_not_running(provider['name'], provider['_id'])
        self.assertFalse(removed)
        failed_to_mark_as_running = is_task_running(provider['name'], provider['_id'], {'minutes': 1})
        self.assertFalse(failed_to_mark_as_running, 'Failed to mark ingest update as running')

        failed_to_mark_as_running = is_task_running(provider['name'], provider['_id'], {'minutes': 1})
        self.assertTrue(failed_to_mark_as_running, 'Ingest update marked as running, possible race condition')

        removed = mark_task_as_not_running(provider['name'], provider['_id'])
        self.assertTrue(removed, 'Failed to mark ingest update as not running.')
예제 #4
0
    def update_overdue_scheduled(self):
        """
        Updates the overdue scheduled content on archive collection.
        """

        logger.info('Updating overdue scheduled content')

        if is_task_running("archive", "update_overdue_scheduled",
                           UPDATE_OVERDUE_SCHEDULED_DEFAULT):
            return

        try:
            now = date_to_str(utcnow())
            items = get_overdue_scheduled_items(now, ARCHIVE)
            item_update = {ITEM_STATE: CONTENT_STATE.PUBLISHED}

            for item in items:
                logger.info(
                    'updating overdue scheduled article with id {} and headline {} -- expired on: {} now: {}'
                    .format(item[config.ID_FIELD], item['headline'],
                            item['publish_schedule'], now))

                superdesk.get_resource_service(ARCHIVE).patch(
                    item[config.ID_FIELD], item_update)
        finally:
            mark_task_as_not_running("archive", "update_overdue_scheduled")
예제 #5
0
    def update_overdue_scheduled(self):
        """
        Updates the overdue scheduled content on published collection.
        """

        logger.info("Updating overdue scheduled content")

        if is_task_running("publish", "update_overdue_scheduled", UPDATE_OVERDUE_SCHEDULED_DEFAULT):
            return

        try:
            now = date_to_str(utcnow())
            items = get_overdue_scheduled_items(now, "published")

            for item in items:
                logger.info(
                    "updating overdue scheduled article with id {} and headline {} -- expired on: {} now: {}".format(
                        item[config.ID_FIELD], item["headline"], item["publish_schedule"], now
                    )
                )

                superdesk.get_resource_service("published").update_published_items(
                    item["item_id"], ITEM_STATE, CONTENT_STATE.PUBLISHED
                )
        finally:
            mark_task_as_not_running("publish", "update_overdue_scheduled")
예제 #6
0
def compare_repo(resource):
    if is_task_running(name=resource, id=1, update_schedule={'seconds': 600}):
        return

    try:
        CompareRepositories().run(resource, ELASTICSEARCH_URL, ELASTICSEARCH_INDEX)
    finally:
        mark_task_as_not_running(name=resource, id=1)
예제 #7
0
    def remove_expired_content(self):
        """
        Removes the expired content from published collection.
        """

        logger.info('Removing expired content from published')

        if is_task_running("publish", "remove_expired", REMOVE_SPIKE_DEFAULT):
            return

        try:
            self.remove_expired_items()
        finally:
            mark_task_as_not_running("publish", "remove_expired")
예제 #8
0
    def remove_expired_content(self):
        """
        Removes the expired content from published collection.
        """

        logger.info('Removing expired content from published')

        if is_task_running("publish", "remove_expired", REMOVE_SPIKE_DEFAULT):
            return

        try:
            self.remove_expired_items()
        finally:
            mark_task_as_not_running("publish", "remove_expired")
예제 #9
0
def publish():
    """
    Fetches items from publish queue as per the configuration,
    calls the transmit function.
    """

    if is_task_running("Transmit", "Articles", UPDATE_SCHEDULE_DEFAULT):
        return

    try:
        items = get_queue_items()

        if items.count() > 0:
            transmit_items(items)
    finally:
        mark_task_as_not_running("Transmit", "Articles")
예제 #10
0
def publish():
    """
    Fetches items from publish queue as per the configuration,
    calls the transmit function.
    """

    if is_task_running("Transmit", "Articles", UPDATE_SCHEDULE_DEFAULT):
        return

    try:
        items = get_queue_items()

        if items.count() > 0:
            transmit_items(items)
    finally:
        mark_task_as_not_running("Transmit", "Articles")
예제 #11
0
def update_provider(provider, rule_set=None, routing_scheme=None):
    """
    Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
    updates the provider.
    """
    if is_task_running(
            provider['name'], provider[superdesk.config.ID_FIELD],
            provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)):
        return

    if provider.get('type') == 'search':
        return

    if not is_updatable(provider):
        return

    try:
        update = {LAST_UPDATED: utcnow()}

        for items in providers[provider.get('type')].update(provider):
            ingest_items(items, provider, rule_set, routing_scheme)
            stats.incr('ingest.ingested_items', len(items))
            if items:
                update[LAST_ITEM_UPDATE] = utcnow()
        ingest_service = superdesk.get_resource_service('ingest_providers')
        ingest_service.system_update(provider[superdesk.config.ID_FIELD],
                                     update, provider)

        if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
            notify_and_add_activity(
                ACTIVITY_EVENT,
                'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
                resource='ingest_providers',
                user_list=ingest_service._get_administrators(),
                name=provider.get('name'),
                last=provider[LAST_ITEM_UPDATE].replace(
                    tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))

        logger.info('Provider {0} updated'.format(
            provider[superdesk.config.ID_FIELD]))
        push_notification('ingest:update',
                          provider_id=str(provider[superdesk.config.ID_FIELD]))
    finally:
        mark_task_as_not_running(provider['name'],
                                 provider[superdesk.config.ID_FIELD])
예제 #12
0
    def remove_expired_content(self):
        logger.info('Removing expired content if spiked')

        if is_task_running("archive", "remove_expired_content", REMOVE_SPIKE_DEFAULT):
            return

        try:
            now = date_to_str(utcnow())
            items = self.get_expired_items(now)

            while items.count() > 0:
                for item in items:
                    logger.info('deleting {} expiry: {} now:{}'.format(item[config.ID_FIELD], item['expiry'], now))
                    superdesk.get_resource_service(ARCHIVE).remove_expired(item)

                items = self.get_expired_items(now)

        finally:
            mark_task_as_not_running("archive", "remove_expired_content")
예제 #13
0
def update_provider(provider, rule_set=None, routing_scheme=None):
    """
    Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
    updates the provider.
    """
    if is_task_running(provider['name'],
                       provider[superdesk.config.ID_FIELD],
                       provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)):
        return

    if provider.get('type') == 'search':
        return

    if not is_updatable(provider):
        return

    try:
        update = {
            LAST_UPDATED: utcnow()
        }

        for items in providers[provider.get('type')].update(provider):
            ingest_items(items, provider, rule_set, routing_scheme)
            stats.incr('ingest.ingested_items', len(items))
            if items:
                update[LAST_ITEM_UPDATE] = utcnow()
        ingest_service = superdesk.get_resource_service('ingest_providers')
        ingest_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)

        if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
            notify_and_add_activity(
                ACTIVITY_EVENT,
                'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
                resource='ingest_providers',
                user_list=ingest_service._get_administrators(),
                name=provider.get('name'),
                last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))

        logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
        push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
    finally:
        mark_task_as_not_running(provider['name'],
                                 provider[superdesk.config.ID_FIELD])
예제 #14
0
def remove_expired_content():
    """
    Celery Task which removes the expired content from published collection.
    """

    logger.info('Removing expired content from published')

    if is_task_running("publish", "remove_expired", UPDATE_SCHEDULE_DEFAULT):
        return

    try:
        now = date_to_str(utcnow())
        items = get_expired_items(now)

        for item in items:
            logger.info('deleting article of type {} with id {} and headline {} -- expired on: {} now: {}'.
                        format(item['type'], item['_id'], item['headline'], item['expiry'], now))

            superdesk.get_resource_service('published').remove_expired(item)
    finally:
        mark_task_as_not_running("publish", "remove_expired")
예제 #15
0
def publish(subscriber, destination):
    """
    Fetches items from publish queue as per the configuration,
    calls the transmit function.
    """
    if is_task_running(destination['name'],
                       subscriber[superdesk.config.ID_FIELD],
                       UPDATE_SCHEDULE_DEFAULT):
        return

    try:
        lookup = {'subscriber_id': subscriber.get('_id'),
                  'destination.name': destination.get('name'),
                  'state': 'pending'}

        items = superdesk.get_resource_service('publish_queue').get(req=None, lookup=lookup)
        if items.count() > 0:
            transmit_items(items, subscriber, destination)
    finally:
        mark_task_as_not_running(destination['name'],
                                 subscriber[superdesk.config.ID_FIELD])
예제 #16
0
    def remove_expired_content(self):
        logger.info('Removing expired content if spiked')

        if is_task_running("archive", "remove_expired_content",
                           REMOVE_SPIKE_DEFAULT):
            return

        try:
            now = date_to_str(utcnow())
            items = self.get_expired_items(now)

            while items.count() > 0:
                for item in items:
                    logger.info('deleting {} expiry: {} now:{}'.format(
                        item[config.ID_FIELD], item['expiry'], now))
                    superdesk.get_resource_service(ARCHIVE).remove_expired(
                        item)

                items = self.get_expired_items(now)

        finally:
            mark_task_as_not_running("archive", "remove_expired_content")
예제 #17
0
    def update_overdue_scheduled(self):
        """
        Updates the overdue scheduled content on archive collection.
        """

        logger.info('Updating overdue scheduled content')

        if is_task_running("archive", "update_overdue_scheduled", UPDATE_OVERDUE_SCHEDULED_DEFAULT):
            return

        try:
            now = date_to_str(utcnow())
            items = get_overdue_scheduled_items(now, ARCHIVE)
            item_update = {ITEM_STATE: CONTENT_STATE.PUBLISHED}

            for item in items:
                logger.info('updating overdue scheduled article with id {} and headline {} -- expired on: {} now: {}'.
                            format(item[config.ID_FIELD], item['headline'], item['publish_schedule'], now))

                superdesk.get_resource_service(ARCHIVE).patch(item['item_id'], item_update)
        finally:
            mark_task_as_not_running("archive", "update_overdue_scheduled")
예제 #18
0
def publish(subscriber, destination):
    """
    Fetches items from publish queue as per the configuration,
    calls the transmit function.
    """
    if is_task_running(destination['name'],
                       subscriber[superdesk.config.ID_FIELD],
                       UPDATE_SCHEDULE_DEFAULT):
        return

    try:
        lookup = {
            'subscriber_id': subscriber.get('_id'),
            'destination.name': destination.get('name'),
            'state': 'pending'
        }

        items = superdesk.get_resource_service('publish_queue').get(
            req=None, lookup=lookup)
        if items.count() > 0:
            transmit_items(items, subscriber, destination)
    finally:
        mark_task_as_not_running(destination['name'],
                                 subscriber[superdesk.config.ID_FIELD])