def move_file(self, file_path, filename, provider, success=True):
"""Move the files from the current directory to the _Processed if successful, else _Error if unsuccessful.
Creates _Processed and _Error directories within current directory if they don't exist.
:param file_path: str - current directory location
:param filename: str - file name in the current directory to move
:param provider: dict - Ingest provider details to which the current directory has been configured
:param success: bool - default value is True. When True moves to _Processed directory else _Error directory.
:raises IngestFileError.folderCreateError() if creation of _Processed or _Error directories fails
:raises IngestFileError.fileMoveError() if failed to move the file pointed by filename
"""
try:
if not os.path.exists(os.path.join(file_path, "_PROCESSED/")):
os.makedirs(os.path.join(file_path, "_PROCESSED/"))
if not os.path.exists(os.path.join(file_path, "_ERROR/")):
os.makedirs(os.path.join(file_path, "_ERROR/"))
except Exception as ex:
raise IngestFileError.folderCreateError(ex, provider)
try:
if success:
shutil.copy2(os.path.join(file_path, filename), os.path.join(file_path, "_PROCESSED/"))
else:
shutil.copy2(os.path.join(file_path, filename), os.path.join(file_path, "_ERROR/"))
except Exception as ex:
raise IngestFileError.fileMoveError(ex, provider)
finally:
os.remove(os.path.join(file_path, filename))
def move_file(self, file_path, filename, provider, success=True):
"""Move the files from the current directory to the _Processed if successful, else _Error if unsuccessful.
Creates _Processed and _Error directories within current directory if they don't exist.
:param file_path: str - current directory location
:param filename: str - file name in the current directory to move
:param provider: dict - Ingest provider details to which the current directory has been configured
:param success: bool - default value is True. When True moves to _Processed directory else _Error directory.
:raises IngestFileError.folderCreateError() if creation of _Processed or _Error directories fails
:raises IngestFileError.fileMoveError() if failed to move the file pointed by filename
"""
try:
if not os.path.exists(os.path.join(file_path, "_PROCESSED/")):
os.makedirs(os.path.join(file_path, "_PROCESSED/"))
if not os.path.exists(os.path.join(file_path, "_ERROR/")):
os.makedirs(os.path.join(file_path, "_ERROR/"))
except Exception as ex:
raise IngestFileError.folderCreateError(ex, provider)
try:
if success:
shutil.copy2(os.path.join(file_path, filename), os.path.join(file_path, "_PROCESSED/"))
else:
shutil.copy2(os.path.join(file_path, filename), os.path.join(file_path, "_ERROR/"))
except Exception as ex:
raise IngestFileError.fileMoveError(ex, provider)
finally:
os.remove(os.path.join(file_path, filename))
def update_provider(provider, rule_set=None, routing_scheme=None):
"""Fetch items from ingest provider, ingest them into Superdesk and update the provider.
:param provider: Ingest Provider data
:param rule_set: Translation Rule Set if one is associated with Ingest Provider.
:param routing_scheme: Routing Scheme if one is associated with Ingest Provider.
"""
lock_name = get_lock_id('ingest', provider['name'],
provider[superdesk.config.ID_FIELD])
if not lock(lock_name, expire=1810):
return
try:
feeding_service = registered_feeding_services[
provider['feeding_service']]
feeding_service = feeding_service.__class__()
update = {LAST_UPDATED: utcnow()}
for items in feeding_service.update(provider, update):
ingest_items(items, provider, feeding_service, rule_set,
routing_scheme)
if items:
update[LAST_ITEM_UPDATE] = utcnow()
# Some Feeding Services update the collection and by this time the _etag might have been changed.
# So it's necessary to fetch it once again. Otherwise, OriginalChangedError is raised.
ingest_provider_service = superdesk.get_resource_service(
'ingest_providers')
provider = ingest_provider_service.find_one(
req=None, _id=provider[superdesk.config.ID_FIELD])
ingest_provider_service.system_update(
provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
admins = superdesk.get_resource_service(
'users').get_users_by_user_type('administrator')
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers',
user_list=admins,
name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(
tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(
provider[superdesk.config.ID_FIELD]))
if LAST_ITEM_UPDATE in update: # Only push a notification if there has been an update
push_notification('ingest:update',
provider_id=str(
provider[superdesk.config.ID_FIELD]))
except Exception as e:
logger.error("Failed to ingest file: {error}".format(error=e))
raise IngestFileError(3000, e, provider)
finally:
unlock(lock_name)
def update_provider(provider, rule_set=None, routing_scheme=None, sync=False):
"""Fetch items from ingest provider, ingest them into Superdesk and update the provider.
:param provider: Ingest Provider data
:param rule_set: Translation Rule Set if one is associated with Ingest Provider.
:param routing_scheme: Routing Scheme if one is associated with Ingest Provider.
:param sync: Running in sync mode from cli.
"""
lock_name = get_lock_id('ingest', provider['name'], provider[superdesk.config.ID_FIELD])
if not lock(lock_name, expire=UPDATE_TTL + 10):
if sync:
logger.error('update is already running for %s', provider['name'])
return
try:
feeding_service = get_feeding_service(provider['feeding_service'])
update = {LAST_UPDATED: utcnow()}
if sync:
provider[LAST_UPDATED] = utcnow() - timedelta(days=9999) # import everything again
generator = feeding_service.update(provider, update)
if isinstance(generator, list):
generator = (items for items in generator)
failed = None
while True:
try:
items = generator.send(failed)
failed = ingest_items(items, provider, feeding_service, rule_set, routing_scheme)
update_last_item_updated(update, items)
except StopIteration:
break
# Some Feeding Services update the collection and by this time the _etag might have been changed.
# So it's necessary to fetch it once again. Otherwise, OriginalChangedError is raised.
ingest_provider_service = superdesk.get_resource_service('ingest_providers')
provider = ingest_provider_service.find_one(req=None, _id=provider[superdesk.config.ID_FIELD])
ingest_provider_service.system_update(provider[superdesk.config.ID_FIELD], update, provider)
if LAST_ITEM_UPDATE not in update and get_is_idle(provider):
admins = superdesk.get_resource_service('users').get_users_by_user_type('administrator')
notify_and_add_activity(
ACTIVITY_EVENT,
'Provider {{name}} has gone strangely quiet. Last activity was on {{last}}',
resource='ingest_providers', user_list=admins, name=provider.get('name'),
last=provider[LAST_ITEM_UPDATE].replace(tzinfo=timezone.utc).astimezone(tz=None).strftime("%c"))
logger.info('Provider {0} updated'.format(provider[superdesk.config.ID_FIELD]))
if LAST_ITEM_UPDATE in update: # Only push a notification if there has been an update
push_notification('ingest:update', provider_id=str(provider[superdesk.config.ID_FIELD]))
except Exception as e:
logger.error("Failed to ingest file: {error}".format(error=e))
raise IngestFileError(3000, e, provider)
finally:
unlock(lock_name)
def test_raise_fileMoveError(self):
with assert_raises(IngestFileError) as error_context:
ex = Exception("Testing fileMoveError")
raise IngestFileError.fileMoveError(ex, self.provider)
exception = error_context.exception
self.assertTrue(exception.code == 3002)
self.assertTrue(exception.message == "Ingest file could not be copied")
self.assertIsNotNone(exception.system_exception)
self.assertEqual(exception.system_exception.args[0], "Testing fileMoveError")
self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
self.assertEqual(self.mock_logger_handler.messages['error'][0],
"IngestFileError Error 3002 - Ingest file could not be copied: "
"Testing fileMoveError on channel TestProvider")
def move_file(self, filepath, filename, provider, success=True):
"""
Move the files from the current directory to the _Processed directory in successful
else _Error if unsuccessful. Creates _Processed and _Error directory within current directory
"""
try:
if not os.path.exists(os.path.join(filepath, "_PROCESSED/")):
os.makedirs(os.path.join(filepath, "_PROCESSED/"))
if not os.path.exists(os.path.join(filepath, "_ERROR/")):
os.makedirs(os.path.join(filepath, "_ERROR/"))
except Exception as ex:
raise IngestFileError.folderCreateError(ex, provider)
try:
if success:
shutil.copy2(os.path.join(filepath, filename), os.path.join(filepath, "_PROCESSED/"))
else:
shutil.copy2(os.path.join(filepath, filename), os.path.join(filepath, "_ERROR/"))
except Exception as ex:
raise IngestFileError.fileMoveError(ex, provider)
finally:
os.remove(os.path.join(filepath, filename))
def test_raise_fileMoveError(self):
with assert_raises(IngestFileError) as error_context:
ex = Exception("Testing fileMoveError")
raise IngestFileError.fileMoveError(ex, self.provider)
exception = error_context.exception
self.assertTrue(exception.code == 3002)
self.assertTrue(exception.message == "Ingest file could not be copied")
self.assertIsNotNone(exception.system_exception)
self.assertEquals(exception.system_exception.args[0],
"Testing fileMoveError")
self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
self.assertEqual(
self.mock_logger_handler.messages['error'][0],
"IngestFileError Error 3002 - Ingest file could not be copied: "
"Testing fileMoveError on channel TestProvider")
def test_raise_folderCreateError(self):
with assert_raises(IngestFileError) as error_context:
ex = Exception("Testing folderCreateError")
raise IngestFileError.folderCreateError(ex, self.provider)
exception = error_context.exception
self.assertTrue(exception.code == 3001)
self.assertTrue(exception.message == "Destination folder could not be created")
self.assertIsNotNone(exception.system_exception)
self.assertEquals(exception.system_exception.args[0], "Testing folderCreateError")
self.assertEqual(len(self.mock_logger_handler.messages["error"]), 1)
self.assertEqual(
self.mock_logger_handler.messages["error"][0],
"IngestFileError Error 3001 - Destination folder could not be created: "
"Testing folderCreateError on channel TestProvider",
)
def test_raise_folderCreateError(self):
with assert_raises(IngestFileError) as error_context:
try:
ex = Exception("Testing folderCreateError")
raise ex
except Exception:
raise IngestFileError.folderCreateError(ex, self.provider)
exception = error_context.exception
self.assertTrue(exception.code == 3001)
self.assertTrue(exception.message == "Destination folder could not be created")
self.assertIsNotNone(exception.system_exception)
self.assertEqual(exception.system_exception.args[0], "Testing folderCreateError")
self.assertEqual(len(self.mock_logger_handler.messages['error']), 1)
self.assertEqual(self.mock_logger_handler.messages['error'][0],
"IngestFileError Error 3001 - Destination folder could not be created: "
"Testing folderCreateError on channel TestProvider")
def _test(self, provider):
path = provider.get('config', {}).get('path', None)
if not os.path.exists(path):
raise IngestFileError.notExistsError()
if not os.path.isdir(path):
raise IngestFileError.isNotDirError()
def _test(self, provider):
path = provider.get('config', {}).get('path', None)
if not os.path.exists(path):
raise IngestFileError.notExistsError()
if not os.path.isdir(path):
raise IngestFileError.isNotDirError()
