def on_created(self, docs):
for doc in docs:
notify_and_add_activity(ACTIVITY_CREATE, 'created Ingest Channel {{name}}', item=doc,
user_list=self._get_administrators(),
name=doc.get('name'))
logger.info("Created Ingest Channel. Data:{}".format(docs))
def on_updated(self, updates, original):
if 'is_closed' not in updates:
do_notification = updates.get('notifications', {})\
.get('on_update', original.get('notifications', {}).get('on_update', True))
notify_and_add_activity(ACTIVITY_UPDATE, 'updated Ingest Channel {{name}}', item=original,
user_list=self._get_administrators()
if do_notification else None,
name=updates.get('name', original.get('name')))
else:
if updates.get('is_closed') != original.get('is_closed', False):
status = ''
do_notification = False
if updates.get('is_closed'):
status = 'closed'
do_notification = updates.get('notifications', {}). \
get('on_close', original.get('notifications', {}).get('on_close', True))
elif not updates.get('is_closed'):
status = 'opened'
do_notification = updates.get('notifications', {}). \
get('on_open', original.get('notifications', {}).get('on_open', True))
notify_and_add_activity(ACTIVITY_EVENT, '{{status}} Ingest Channel {{name}}', item=original,
user_list=self._get_administrators()
if do_notification else None,
name=updates.get('name', original.get('name')),
status=status)
logger.info("Updated Ingest Channel. Data: {}".format(updates))
def on_updated(self, updates, original):
do_notification = updates.get('notifications', {})\
.get('on_update', original.get('notifications', {}).get('on_update', True))
notify_and_add_activity(ACTIVITY_UPDATE, 'updated Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator')
if do_notification else None,
name=updates.get('name', original.get('name')),
provider_id=original.get('_id'))
if updates.get('is_closed', False) != original.get('is_closed', False):
status = ''
do_notification = False
if updates.get('is_closed'):
status = 'closed'
do_notification = updates.get('notifications', {}). \
get('on_close', original.get('notifications', {}).get('on_close', True))
elif not updates.get('is_closed'):
status = 'opened'
do_notification = updates.get('notifications', {}). \
get('on_open', original.get('notifications', {}).get('on_open', True))
notify_and_add_activity(ACTIVITY_EVENT, '{{status}} Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator')
if do_notification else None,
name=updates.get('name', original.get('name')),
status=status, provider_id=original.get('_id'))
push_notification('ingest_provider:update', provider_id=str(original.get('_id')))
logger.info("Updated Ingest Channel. Data: {}".format(updates))
def update_provider(provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
"""
if ingest_for_provider_is_already_running(provider):
return
try:
update = {
LAST_UPDATED: utcnow()
}
for items in providers[provider.get('type')].update(provider):
ingest_items(items, provider, rule_set, routing_scheme)
stats.incr('ingest.ingested_items', len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
ingest_service = superdesk.get_resource_service('ingest_providers')
ingest_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=ingest_service._get_administrators(),
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
finally:
mark_provider_as_not_running(provider)
def on_updated(self, updates, original):
do_notification = updates.get('notifications', {})\
.get('on_update', original.get('notifications', {}).get('on_update', True))
notify_and_add_activity(ACTIVITY_UPDATE, 'updated Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator')
if do_notification else None,
name=updates.get('name', original.get('name')),
provider_id=original.get('_id'))
if updates.get('is_closed', False) != original.get('is_closed', False):
status = ''
do_notification = False
if updates.get('is_closed'):
status = 'closed'
do_notification = updates.get('notifications', {}). \
get('on_close', original.get('notifications', {}).get('on_close', True))
elif not updates.get('is_closed'):
status = 'opened'
do_notification = updates.get('notifications', {}). \
get('on_open', original.get('notifications', {}).get('on_open', True))
notify_and_add_activity(ACTIVITY_EVENT, '{{status}} Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator')
if do_notification else None,
name=updates.get('name', original.get('name')),
status=status, provider_id=original.get('_id'))
push_notification('ingest_provider:update', provider_id=str(original.get('_id')))
logger.info("Updated Ingest Channel. Data: {}".format(updates))
def on_updated(self, updates, original):
if 'is_closed' not in updates:
do_notification = updates.get('notifications', {})\
.get('on_update', original.get('notifications', {}).get('on_update', True))
notify_and_add_activity(ACTIVITY_UPDATE,
'updated Ingest Channel {{name}}',
item=original,
user_list=self._get_administrators()
if do_notification else None,
name=updates.get('name',
original.get('name')))
else:
if updates.get('is_closed') != original.get('is_closed', False):
status = ''
do_notification = False
if updates.get('is_closed'):
status = 'closed'
do_notification = updates.get('notifications', {}). \
get('on_close', original.get('notifications', {}).get('on_close', True))
elif not updates.get('is_closed'):
status = 'opened'
do_notification = updates.get('notifications', {}). \
get('on_open', original.get('notifications', {}).get('on_open', True))
notify_and_add_activity(ACTIVITY_EVENT,
'{{status}} Ingest Channel {{name}}',
item=original,
user_list=self._get_administrators()
if do_notification else None,
name=updates.get(
'name', original.get('name')),
status=status)
logger.info("Updated Ingest Channel. Data: {}".format(updates))
def update_provider(provider, rule_set=None, routing_scheme=None):
"""Fetch items from ingest provider, ingest them into Superdesk and update the provider.
:param provider: Ingest Provider data
:param rule_set: Translation Rule Set if one is associated with Ingest Provider.
:param routing_scheme: Routing Scheme if one is associated with Ingest Provider.
"""
lock_name = get_lock_id('ingest', provider['name'],
provider[superdesk.config.ID_FIELD])
if not lock(lock_name, expire=1810):
return
try:
feeding_service = registered_feeding_services[
provider['feeding_service']]
feeding_service = feeding_service.__class__()
update = {LAST_UPDATED: utcnow()}
for items in feeding_service.update(provider, update):
ingest_items(items, provider, feeding_service, rule_set,
routing_scheme)
if items:
update[LAST_ITEM_UPDATE] = utcnow()
# Some Feeding Services update the collection and by this time the _etag might have been changed.
# So it's necessary to fetch it once again. Otherwise, OriginalChangedError is raised.
ingest_provider_service = superdesk.get_resource_service(
'ingest_providers')
provider = ingest_provider_service.find_one(
req=None, _id=provider[superdesk.config.ID_FIELD])
ingest_provider_service.system_update(
provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
admins = superdesk.get_resource_service(
'users').get_users_by_user_type('administrator')
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=admins,
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(
tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(
provider[superdesk.config.ID_FIELD]))
if LAST_ITEM_UPDATE in update: # Only push a notification if there has been an update
push_notification('ingest:update',
provider_id=str(
provider[superdesk.config.ID_FIELD]))
except Exception as e:
logger.error("Failed to ingest file: {error}".format(error=e))
raise IngestFileError(3000, e, provider)
finally:
unlock(lock_name)
def on_deleted(self, doc):
notify_and_add_activity(ACTIVITY_DELETE,
'deleted Ingest Channel {{name}}',
item=doc,
user_list=self._get_administrators(),
name=doc.get('name'))
logger.info("Deleted Ingest Channel. Data: {}".format(doc))
def on_created(self, docs):
for doc in docs:
notify_and_add_activity(ACTIVITY_CREATE, 'Created Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator'),
name=doc.get('name'), provider_id=doc.get('_id'))
push_notification('ingest_provider:create', provider_id=str(doc.get('_id')))
logger.info("Created Ingest Channel. Data:{}".format(docs))
def on_created(self, docs):
for doc in docs:
notify_and_add_activity(ACTIVITY_CREATE, 'Created Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator'),
name=doc.get('name'), provider_id=doc.get('_id'))
push_notification('ingest_provider:create', provider_id=str(doc.get('_id')))
logger.info("Created Ingest Channel. Data:{}".format(docs))
def update_provider(provider, rule_set=None, routing_scheme=None, sync=False):
"""Fetch items from ingest provider, ingest them into Superdesk and update the provider.
:param provider: Ingest Provider data
:param rule_set: Translation Rule Set if one is associated with Ingest Provider.
:param routing_scheme: Routing Scheme if one is associated with Ingest Provider.
:param sync: Running in sync mode from cli.
"""
lock_name = get_lock_id('ingest', provider['name'], provider[superdesk.config.ID_FIELD])
if not lock(lock_name, expire=UPDATE_TTL + 10):
if sync:
logger.error('update is already running for %s', provider['name'])
return
try:
feeding_service = get_feeding_service(provider['feeding_service'])
update = {LAST_UPDATED: utcnow()}
if sync:
provider[LAST_UPDATED] = utcnow() - timedelta(days=9999) # import everything again
generator = feeding_service.update(provider, update)
if isinstance(generator, list):
generator = (items for items in generator)
failed = None
while True:
try:
items = generator.send(failed)
failed = ingest_items(items, provider, feeding_service, rule_set, routing_scheme)
update_last_item_updated(update, items)
except StopIteration:
break
# Some Feeding Services update the collection and by this time the _etag might have been changed.
# So it's necessary to fetch it once again. Otherwise, OriginalChangedError is raised.
ingest_provider_service = superdesk.get_resource_service('ingest_providers')
provider = ingest_provider_service.find_one(req=None, _id=provider[superdesk.config.ID_FIELD])
ingest_provider_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
admins = superdesk.get_resource_service('users').get_users_by_user_type('administrator')
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers', user_list=admins, name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
if LAST_ITEM_UPDATE in update: # Only push a notification if there has been an update
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
except Exception as e:
logger.error("Failed to ingest file: {error}".format(error=e))
raise IngestFileError(3000, e, provider)
finally:
unlock(lock_name)
def on_created(self, docs):
for doc in docs:
notify_and_add_activity(ACTIVITY_CREATE,
'created Ingest Channel {{name}}',
item=doc,
user_list=self._get_administrators(),
name=doc.get('name'))
logger.info("Created Ingest Channel. Data:{}".format(docs))
def on_deleted(self, doc):
"""
Overriding to send notification and record activity about channel deletion.
"""
notify_and_add_activity(ACTIVITY_DELETE, 'Deleted Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self._get_administrators(),
name=doc.get('name'), provider_id=doc.get(config.ID_FIELD))
push_notification('ingest_provider:delete', provider_id=str(doc.get(config.ID_FIELD)))
logger.info("Deleted Ingest Channel. Data:{}".format(doc))
def update_provider(self, provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
:param self:
:type self:
:param provider: Ingest Provider Details
:type provider: dict :py:class:`superdesk.io.ingest_provider_model.IngestProviderResource`
:param rule_set: Translation Rule Set if one is associated with Ingest Provider.
:type rule_set: dict :py:class:`apps.rules.rule_sets.RuleSetsResource`
:param routing_scheme: Routing Scheme if one is associated with Ingest Provider.
:type routing_scheme: dict :py:class:`apps.rules.routing_rules.RoutingRuleSchemeResource`
"""
lock_name = get_lock_id('ingest', provider['name'], provider[superdesk.config.ID_FIELD])
host_name = get_host_id(self)
if not lock(lock_name, host_name, expire=1800):
return
try:
feeding_service = registered_feeding_services[provider['feeding_service']]
feeding_service = feeding_service.__class__()
update = {LAST_UPDATED: utcnow()}
for items in feeding_service.update(provider):
ingest_items(items, provider, feeding_service, rule_set, routing_scheme)
stats.incr('ingest.ingested_items', len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
# Some Feeding Services update the collection and by this time the _etag might have been changed.
# So it's necessary to fetch it once again. Otherwise, OriginalChangedError is raised.
ingest_provider_service = superdesk.get_resource_service('ingest_providers')
provider = ingest_provider_service.find_one(req=None, _id=provider[superdesk.config.ID_FIELD])
ingest_provider_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
admins = superdesk.get_resource_service('users').get_users_by_user_type('administrator')
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers', user_list=admins, name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
if LAST_ITEM_UPDATE in update: # Only push a notification if there has been an update
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
finally:
unlock(lock_name, host_name)
def on_deleted(self, doc):
"""
Overriding to send notification and record activity about channel deletion.
"""
notify_and_add_activity(ACTIVITY_DELETE, 'Deleted Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator'),
name=doc.get('name'), provider_id=doc.get(config.ID_FIELD))
push_notification('ingest_provider:delete', provider_id=str(doc.get(config.ID_FIELD)))
get_resource_service('sequences').delete(lookup={
'key': 'ingest_providers_{_id}'.format(_id=doc[config.ID_FIELD])
})
logger.info("Deleted Ingest Channel. Data:{}".format(doc))
def on_deleted(self, doc):
"""
Overriding to send notification and record activity about channel deletion.
"""
notify_and_add_activity(ACTIVITY_DELETE, 'Deleted Ingest Channel {{name}}',
self.datasource, item=None,
user_list=self.user_service.get_users_by_user_type('administrator'),
name=doc.get('name'), provider_id=doc.get(config.ID_FIELD))
push_notification('ingest_provider:delete', provider_id=str(doc.get(config.ID_FIELD)))
get_resource_service('sequences').delete(lookup={
'key': 'ingest_providers_{_id}'.format(_id=doc[config.ID_FIELD])
})
logger.info("Deleted Ingest Channel. Data:{}".format(doc))
def update_provider(provider, rule_set=None, routing_scheme=None):
"""Fetch items from ingest provider, ingest them into Superdesk and update the provider.
:param provider: Ingest Provider data
:param rule_set: Translation Rule Set if one is associated with Ingest Provider.
:param routing_scheme: Routing Scheme if one is associated with Ingest Provider.
"""
lock_name = get_lock_id('ingest', provider['name'], provider[superdesk.config.ID_FIELD])
if not lock(lock_name, expire=1810):
return
try:
feeding_service = registered_feeding_services[provider['feeding_service']]
feeding_service = feeding_service.__class__()
update = {LAST_UPDATED: utcnow()}
for items in feeding_service.update(provider, update):
ingest_items(items, provider, feeding_service, rule_set, routing_scheme)
if items:
last_item_update = max(
[item['versioncreated'] for item in items if item.get('versioncreated')],
default=utcnow()
)
if not update.get(LAST_ITEM_UPDATE) or update[LAST_ITEM_UPDATE] < last_item_update:
update[LAST_ITEM_UPDATE] = last_item_update
# Some Feeding Services update the collection and by this time the _etag might have been changed.
# So it's necessary to fetch it once again. Otherwise, OriginalChangedError is raised.
ingest_provider_service = superdesk.get_resource_service('ingest_providers')
provider = ingest_provider_service.find_one(req=None, _id=provider[superdesk.config.ID_FIELD])
ingest_provider_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
admins = superdesk.get_resource_service('users').get_users_by_user_type('administrator')
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers', user_list=admins, name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
if LAST_ITEM_UPDATE in update: # Only push a notification if there has been an update
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
except Exception as e:
logger.error("Failed to ingest file: {error}".format(error=e))
raise IngestFileError(3000, e, provider)
finally:
unlock(lock_name)
def handle_mark_user_notifications(self, updates, original):
"""Notify user when item is marked or unmarked
:param updates: updates to item that should be saved
:param original: original item version before update
"""
orig_marked_user = original.get('marked_for_user', None)
new_marked_user = updates.get('marked_for_user', None)
by_user = get_user().get('display_name', get_user().get('username'))
if new_marked_user:
marked_user = get_resource_service('users').find_one(req=None, _id=new_marked_user)
marked_for_user = marked_user.get('display_name', marked_user.get('username'))
if orig_marked_user and new_marked_user is None:
# sent when unmarking user from item
user_list = [{'_id': orig_marked_user}]
message = 'Item "{headline}" has been unmarked by {by_user}.'.format(
headline=original.get('headline', original.get('slugline', 'item')),
by_user=by_user)
notify_and_add_activity('item:unmarked', message,
resource=self.datasource, item=original,
user_list=user_list)
# send separate notification for markForUser extension
push_notification('item:unmarked',
item_id=original.get(config.ID_FIELD),
user_list=user_list,
extension='markForUser')
else:
# sent when mark item for user or mark to another user
if new_marked_user and orig_marked_user and new_marked_user != orig_marked_user:
user_list = [{'_id': new_marked_user}, {'_id': orig_marked_user}]
else:
user_list = [{'_id': new_marked_user}]
message = 'Item "{headline}" has been marked for {for_user} by {by_user}.'.format(
headline=original.get('headline', original.get('slugline', 'item')),
for_user=marked_for_user,
by_user=by_user)
notify_and_add_activity('item:marked', message,
resource=self.datasource, item=original,
user_list=user_list,
marked_for_user=marked_for_user)
# send separate notification for markForUser extension
push_notification('item:marked',
item_id=original.get(config.ID_FIELD),
user_list=user_list,
extension='markForUser')
def update_provider(self, provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
"""
if provider.get('type') == 'search':
return
if not is_updatable(provider):
return
lock_name = get_lock_id('ingest', provider['name'],
provider[superdesk.config.ID_FIELD])
host_name = get_host_id(self)
if not lock(lock_name, host_name, expire=1800):
return
try:
update = {LAST_UPDATED: utcnow()}
for items in providers[provider.get('type')].update(provider):
ingest_items(items, provider, rule_set, routing_scheme)
stats.incr('ingest.ingested_items', len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
ingest_service = superdesk.get_resource_service('ingest_providers')
ingest_service.system_update(provider[superdesk.config.ID_FIELD],
update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=ingest_service._get_administrators(),
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(
tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(
provider[superdesk.config.ID_FIELD]))
# Only push a notification if there has been an update
if LAST_ITEM_UPDATE in update:
push_notification('ingest:update',
provider_id=str(
provider[superdesk.config.ID_FIELD]))
finally:
unlock(lock_name, host_name)
def on_updated(self, updates, original):
do_notification = updates.get("notifications", {}).get(
"on_update",
original.get("notifications", {}).get("on_update", True))
notify_and_add_activity(
ACTIVITY_UPDATE,
"updated Ingest Channel {{name}}",
self.datasource,
item=None,
user_list=self.user_service.get_users_by_user_type("administrator")
if do_notification else None,
name=updates.get("name", original.get("name")),
provider_id=original.get("_id"),
)
if updates.get("is_closed", False) != original.get("is_closed", False):
status = ""
do_notification = False
if updates.get("is_closed"):
status = "closed"
do_notification = updates.get("notifications", {}).get(
"on_close",
original.get("notifications", {}).get("on_close", True))
elif not updates.get("is_closed"):
status = "opened"
do_notification = updates.get("notifications", {}).get(
"on_open",
original.get("notifications", {}).get("on_open", True))
notify_and_add_activity(
ACTIVITY_EVENT,
"{{status}} Ingest Channel {{name}}",
self.datasource,
item=None,
user_list=self.user_service.get_users_by_user_type(
"administrator") if do_notification else None,
name=updates.get("name", original.get("name")),
status=status,
provider_id=original.get("_id"),
)
push_notification("ingest_provider:update",
provider_id=str(original.get("_id")))
logger.info("Updated Ingest Channel. Data: {}".format(updates))
def update_provider(self, provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
"""
if provider.get('type') == 'search':
return
if not is_updatable(provider):
return
lock_name = get_lock_id('ingest', provider['name'], provider[superdesk.config.ID_FIELD])
host_name = get_host_id(self)
if not lock(lock_name, host_name, expire=1800):
return
try:
update = {
LAST_UPDATED: utcnow()
}
for items in providers[provider.get('type')].update(provider):
ingest_items(items, provider, rule_set, routing_scheme)
stats.incr('ingest.ingested_items', len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
ingest_service = superdesk.get_resource_service('ingest_providers')
ingest_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=ingest_service._get_administrators(),
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
# Only push a notification if there has been an update
if LAST_ITEM_UPDATE in update:
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
finally:
unlock(lock_name, host_name)
def on_updated(self, updates, original):
do_notification = updates.get("notifications", {}).get(
"on_update", original.get("notifications", {}).get("on_update", True)
)
notify_and_add_activity(
ACTIVITY_UPDATE,
"updated Ingest Channel {{name}}",
self.datasource,
item=None,
user_list=self.user_service.get_users_by_user_type("administrator") if do_notification else None,
name=updates.get("name", original.get("name")),
provider_id=original.get("_id"),
)
if updates.get("is_closed", False) != original.get("is_closed", False):
status = ""
do_notification = False
if updates.get("is_closed"):
status = "closed"
do_notification = updates.get("notifications", {}).get(
"on_close", original.get("notifications", {}).get("on_close", True)
)
elif not updates.get("is_closed"):
status = "opened"
do_notification = updates.get("notifications", {}).get(
"on_open", original.get("notifications", {}).get("on_open", True)
)
notify_and_add_activity(
ACTIVITY_EVENT,
"{{status}} Ingest Channel {{name}}",
self.datasource,
item=None,
user_list=self.user_service.get_users_by_user_type("administrator") if do_notification else None,
name=updates.get("name", original.get("name")),
status=status,
provider_id=original.get("_id"),
)
push_notification("ingest_provider:update", provider_id=str(original.get("_id")))
logger.info("Updated Ingest Channel. Data: {}".format(updates))
def update_provider(provider, rule_set=None, routing_scheme=None):
"""
Fetches items from ingest provider as per the configuration, ingests them into Superdesk and
updates the provider.
"""
if is_task_running(
provider["name"], provider[superdesk.config.ID_FIELD], provider.get("update_schedule", UPDATE_SCHEDULE_DEFAULT)
):
return
if provider.get("type") == "search":
return
if not is_updatable(provider):
return
try:
update = {LAST_UPDATED: utcnow()}
for items in providers[provider.get("type")].update(provider):
ingest_items(items, provider, rule_set, routing_scheme)
stats.incr("ingest.ingested_items", len(items))
if items:
update[LAST_ITEM_UPDATE] = utcnow()
ingest_service = superdesk.get_resource_service("ingest_providers")
ingest_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
notify_and_add_activity(
ACTIVITY_EVENT,
"Provider {{name}} has gone strangely quiet. Last activity was on {{last}}",
resource="ingest_providers",
user_list=ingest_service._get_administrators(),
name=provider.get("name"),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"),
)
logger.info("Provider {0} updated".format(provider[superdesk.config.ID_FIELD]))
push_notification("ingest:update", provider_id=str(provider[superdesk.config.ID_FIELD]))
finally:
mark_task_as_not_running(provider["name"], provider[superdesk.config.ID_FIELD])
def on_deleted(self, doc):
notify_and_add_activity(ACTIVITY_DELETE, 'deleted Ingest Channel {{name}}', item=doc,
user_list=self._get_administrators(),
name=doc.get('name'))
logger.info("Deleted Ingest Channel. Data: {}".format(doc))