def test_the_second_update_fails_if_already_running(self):
provider = {
"_id": "abc",
"name": "test provider",
"update_schedule": {
"minutes": 1
}
}
removed = mark_task_as_not_running(provider["name"], provider["_id"])
self.assertFalse(removed)
failed_to_mark_as_running = is_task_running(provider["name"],
provider["_id"],
{"minutes": 1})
self.assertFalse(failed_to_mark_as_running,
"Failed to mark ingest update as running")
failed_to_mark_as_running = is_task_running(provider["name"],
provider["_id"],
{"minutes": 1})
self.assertTrue(
failed_to_mark_as_running,
"Ingest update marked as running, possible race condition")
removed = mark_task_as_not_running(provider["name"], provider["_id"])
self.assertTrue(removed,
"Failed to mark ingest update as not running.")
def test_the_second_update_fails_if_already_running(self):
provider = {
'_id': 'abc',
'name': 'test provider',
'update_schedule': {
'minutes': 1
}
}
removed = mark_task_as_not_running(provider['name'], provider['_id'])
self.assertFalse(removed)
failed_to_mark_as_running = is_task_running(provider['name'],
provider['_id'],
{'minutes': 1})
self.assertFalse(failed_to_mark_as_running,
'Failed to mark ingest update as running')
failed_to_mark_as_running = is_task_running(provider['name'],
provider['_id'],
{'minutes': 1})
self.assertTrue(
failed_to_mark_as_running,
'Ingest update marked as running, possible race condition')
removed = mark_task_as_not_running(provider['name'], provider['_id'])
self.assertTrue(removed,
'Failed to mark ingest update as not running.')
def test_the_second_update_fails_if_already_running(self):
provider = {'_id': 'abc', 'name': 'test provider', 'update_schedule': {'minutes': 1}}
removed = mark_task_as_not_running(provider['name'], provider['_id'])
self.assertFalse(removed)
failed_to_mark_as_running = is_task_running(provider['name'], provider['_id'], {'minutes': 1})
self.assertFalse(failed_to_mark_as_running, 'Failed to mark ingest update as running')
failed_to_mark_as_running = is_task_running(provider['name'], provider['_id'], {'minutes': 1})
self.assertTrue(failed_to_mark_as_running, 'Ingest update marked as running, possible race condition')
removed = mark_task_as_not_running(provider['name'], provider['_id'])
self.assertTrue(removed, 'Failed to mark ingest update as not running.')
def update_overdue_scheduled(self):
"""
Updates the overdue scheduled content on archive collection.
"""
logger.info('Updating overdue scheduled content')
if is_task_running("archive", "update_overdue_scheduled",
UPDATE_OVERDUE_SCHEDULED_DEFAULT):
return
try:
now = date_to_str(utcnow())
items = get_overdue_scheduled_items(now, ARCHIVE)
item_update = {ITEM_STATE: CONTENT_STATE.PUBLISHED}
for item in items:
logger.info(
'updating overdue scheduled article with id {} and headline {} -- expired on: {} now: {}'
.format(item[config.ID_FIELD], item['headline'],
item['publish_schedule'], now))
superdesk.get_resource_service(ARCHIVE).patch(
item[config.ID_FIELD], item_update)
finally:
mark_task_as_not_running("archive", "update_overdue_scheduled")
def update_overdue_scheduled(self):
"""
Updates the overdue scheduled content on published collection.
"""
logger.info("Updating overdue scheduled content")
if is_task_running("publish", "update_overdue_scheduled", UPDATE_OVERDUE_SCHEDULED_DEFAULT):
return
try:
now = date_to_str(utcnow())
items = get_overdue_scheduled_items(now, "published")
for item in items:
logger.info(
"updating overdue scheduled article with id {} and headline {} -- expired on: {} now: {}".format(
item[config.ID_FIELD], item["headline"], item["publish_schedule"], now
)
)
superdesk.get_resource_service("published").update_published_items(
item["item_id"], ITEM_STATE, CONTENT_STATE.PUBLISHED
)
finally:
mark_task_as_not_running("publish", "update_overdue_scheduled")
def compare_repo(resource):
if is_task_running(name=resource, id=1, update_schedule={'seconds': 600}):
return
try:
CompareRepositories().run(resource, ELASTICSEARCH_URL, ELASTICSEARCH_INDEX)
finally:
mark_task_as_not_running(name=resource, id=1)
def remove_expired_content(self):
"""
Removes the expired content from published collection.
"""
logger.info('Removing expired content from published')
if is_task_running("publish", "remove_expired", REMOVE_SPIKE_DEFAULT):
return
try:
self.remove_expired_items()
finally:
mark_task_as_not_running("publish", "remove_expired")
def remove_expired_content(self):
"""
Removes the expired content from published collection.
"""
logger.info('Removing expired content from published')
if is_task_running("publish", "remove_expired", REMOVE_SPIKE_DEFAULT):
return
try:
self.remove_expired_items()
finally:
mark_task_as_not_running("publish", "remove_expired")
def publish():
"""
Fetches items from publish queue as per the configuration,
calls the transmit function.
"""
if is_task_running("Transmit", "Articles", UPDATE_SCHEDULE_DEFAULT):
return
try:
items = get_queue_items()
if items.count() > 0:
transmit_items(items)
finally:
mark_task_as_not_running("Transmit", "Articles")
def publish():
"""
Fetches items from publish queue as per the configuration,
calls the transmit function.
"""
if is_task_running("Transmit", "Articles", UPDATE_SCHEDULE_DEFAULT):
return
try:
items = get_queue_items()
if items.count() > 0:
transmit_items(items)
finally:
mark_task_as_not_running("Transmit", "Articles")
def update_provider(provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
"""
if is_task_running(
provider['name'], provider[superdesk.config.ID_FIELD],
provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)):
return
if provider.get('type') == 'search':
return
if not is_updatable(provider):
return
try:
update = {LAST_UPDATED: utcnow()}
for items in providers[provider.get('type')].update(provider):
ingest_items(items, provider, rule_set, routing_scheme)
stats.incr('ingest.ingested_items', len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
ingest_service = superdesk.get_resource_service('ingest_providers')
ingest_service.system_update(provider[superdesk.config.ID_FIELD],
update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=ingest_service._get_administrators(),
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(
tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(
provider[superdesk.config.ID_FIELD]))
push_notification('ingest:update',
provider_id=str(provider[superdesk.config.ID_FIELD]))
finally:
mark_task_as_not_running(provider['name'],
provider[superdesk.config.ID_FIELD])
def remove_expired_content(self):
logger.info('Removing expired content if spiked')
if is_task_running("archive", "remove_expired_content", REMOVE_SPIKE_DEFAULT):
return
try:
now = date_to_str(utcnow())
items = self.get_expired_items(now)
while items.count() > 0:
for item in items:
logger.info('deleting {} expiry: {} now:{}'.format(item[config.ID_FIELD], item['expiry'], now))
superdesk.get_resource_service(ARCHIVE).remove_expired(item)
items = self.get_expired_items(now)
finally:
mark_task_as_not_running("archive", "remove_expired_content")
def update_provider(provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
"""
if is_task_running(provider['name'],
provider[superdesk.config.ID_FIELD],
provider.get('update_schedule', UPDATE_SCHEDULE_DEFAULT)):
return
if provider.get('type') == 'search':
return
if not is_updatable(provider):
return
try:
update = {
LAST_UPDATED: utcnow()
}
for items in providers[provider.get('type')].update(provider):
ingest_items(items, provider, rule_set, routing_scheme)
stats.incr('ingest.ingested_items', len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
ingest_service = superdesk.get_resource_service('ingest_providers')
ingest_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=ingest_service._get_administrators(),
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
finally:
mark_task_as_not_running(provider['name'],
provider[superdesk.config.ID_FIELD])
def remove_expired_content():
"""
Celery Task which removes the expired content from published collection.
"""
logger.info('Removing expired content from published')
if is_task_running("publish", "remove_expired", UPDATE_SCHEDULE_DEFAULT):
return
try:
now = date_to_str(utcnow())
items = get_expired_items(now)
for item in items:
logger.info('deleting article of type {} with id {} and headline {} -- expired on: {} now: {}'.
format(item['type'], item['_id'], item['headline'], item['expiry'], now))
superdesk.get_resource_service('published').remove_expired(item)
finally:
mark_task_as_not_running("publish", "remove_expired")
def publish(subscriber, destination):
"""
Fetches items from publish queue as per the configuration,
calls the transmit function.
"""
if is_task_running(destination['name'],
subscriber[superdesk.config.ID_FIELD],
UPDATE_SCHEDULE_DEFAULT):
return
try:
lookup = {'subscriber_id': subscriber.get('_id'),
'destination.name': destination.get('name'),
'state': 'pending'}
items = superdesk.get_resource_service('publish_queue').get(req=None, lookup=lookup)
if items.count() > 0:
transmit_items(items, subscriber, destination)
finally:
mark_task_as_not_running(destination['name'],
subscriber[superdesk.config.ID_FIELD])
def remove_expired_content(self):
logger.info('Removing expired content if spiked')
if is_task_running("archive", "remove_expired_content",
REMOVE_SPIKE_DEFAULT):
return
try:
now = date_to_str(utcnow())
items = self.get_expired_items(now)
while items.count() > 0:
for item in items:
logger.info('deleting {} expiry: {} now:{}'.format(
item[config.ID_FIELD], item['expiry'], now))
superdesk.get_resource_service(ARCHIVE).remove_expired(
item)
items = self.get_expired_items(now)
finally:
mark_task_as_not_running("archive", "remove_expired_content")
def update_overdue_scheduled(self):
"""
Updates the overdue scheduled content on archive collection.
"""
logger.info('Updating overdue scheduled content')
if is_task_running("archive", "update_overdue_scheduled", UPDATE_OVERDUE_SCHEDULED_DEFAULT):
return
try:
now = date_to_str(utcnow())
items = get_overdue_scheduled_items(now, ARCHIVE)
item_update = {ITEM_STATE: CONTENT_STATE.PUBLISHED}
for item in items:
logger.info('updating overdue scheduled article with id {} and headline {} -- expired on: {} now: {}'.
format(item[config.ID_FIELD], item['headline'], item['publish_schedule'], now))
superdesk.get_resource_service(ARCHIVE).patch(item['item_id'], item_update)
finally:
mark_task_as_not_running("archive", "update_overdue_scheduled")
def publish(subscriber, destination):
"""
Fetches items from publish queue as per the configuration,
calls the transmit function.
"""
if is_task_running(destination['name'],
subscriber[superdesk.config.ID_FIELD],
UPDATE_SCHEDULE_DEFAULT):
return
try:
lookup = {
'subscriber_id': subscriber.get('_id'),
'destination.name': destination.get('name'),
'state': 'pending'
}
items = superdesk.get_resource_service('publish_queue').get(
req=None, lookup=lookup)
if items.count() > 0:
transmit_items(items, subscriber, destination)
finally:
mark_task_as_not_running(destination['name'],
subscriber[superdesk.config.ID_FIELD])