def test_allowed_suffix_json(self):
"""Check that json files are allowed for ingestion."""
service = ftp.FTPFeedingService()
self.assertFalse(service._is_allowed('foo.jpg'))
self.assertTrue(service._is_allowed('foo.xml'))
self.assertTrue(service._is_allowed('foo.json'))
self.assertTrue(service._is_allowed('foo.JSON'))
self.assertFalse(service._is_allowed('foojson'))
self.assertFalse(service._is_allowed('foo.json.tar.gz'))
def test_allowed_suffix_json(self):
"""Check that json files are allowed for ingestion."""
service = ftp.FTPFeedingService()
allowed = ftp.FTPFeedingService.ALLOWED_EXT_DEFAULT
self.assertFalse(service._is_allowed('foo.jpg', allowed))
self.assertTrue(service._is_allowed('foo.xml', allowed))
self.assertTrue(service._is_allowed('foo.json', allowed))
self.assertTrue(service._is_allowed('foo.JSON', allowed))
self.assertFalse(service._is_allowed('foojson', allowed))
self.assertFalse(service._is_allowed('foo.json.tar.gz', allowed))
def test_move_ingested(self, ftp_connect):
"""Check that ingested file is moved if "move" is set
feature requested in SDESK-468
"""
provider = PROVIDER.copy()
service = ftp.FTPFeedingService()
service._update(provider, {})
mock_ftp = ftp_connect.return_value.__enter__.return_value
mock_ftp.rename.assert_called_once_with('filename.xml', 'dest_move/filename.xml')
def test_move_error(self, ftp_connect):
"""Check that error on ingestion moves item if "move_path_error" is set
feature requested in SDESK-1452
"""
provider = PROVIDER.copy()
service = ftp.FTPFeedingService()
service._update(provider, {})
mock_ftp = ftp_connect.return_value.__enter__.return_value
mock_ftp.rename.assert_called_once_with('filename.xml', 'error/filename.xml')
def test_move_ingested_no_move(self, ftp_connect):
"""Check that ingested file is not moved if "move" is not set
feature requested in SDESK-468
"""
provider = copy.deepcopy(PROVIDER)
provider['config']['move'] = False
service = ftp.FTPFeedingService()
service._update(provider, {})
mock_ftp = ftp_connect.return_value.__enter__.return_value
mock_ftp.rename.assert_not_called()
def test_move_error_default(self, ftp_connect):
"""Check that error on ingestion use default path if "move_path_error" is empty string
feature requested in SDESK-1452
"""
provider = PROVIDER.copy()
provider['config']['move_path_error'] = ""
service = ftp.FTPFeedingService()
service._update(provider, {})
mock_ftp = ftp_connect.return_value.__enter__.return_value
dest_path = os.path.join(ftp.DEFAULT_FAILURE_PATH, "filename.xml")
mock_ftp.rename.assert_called_once_with('filename.xml', dest_path)
def test_move_ingested_error(self, ftp_connect, *args):
"""Check that ingested file is moved to error path if ingest fails"""
provider = copy.deepcopy(PROVIDER)
provider["config"]["ftp_move_path"] = ""
service = ftp.FTPFeedingService()
service._is_empty = mock.MagicMock(return_value=False)
ingest_items(service.update(provider, {}), False)
mock_ftp = ftp_connect.return_value.__enter__.return_value
self.assertEqual(mock_ftp.rename.call_count, len(FakeFTP.files))
for i, call in enumerate(mock_ftp.rename.call_args_list):
self.assertEqual(call[0], (FakeFTP.files[i][0], "{}/{}".format("error", FakeFTP.files[i][0])))
def test_move_error(self, ftp_connect):
"""Check that error on ingestion moves item if "move_path_error" is set
feature requested in SDESK-1452
"""
provider = copy.deepcopy(PROVIDER)
service = ftp.FTPFeedingService()
service._is_empty = mock.MagicMock(return_value=False)
ingest_items(service.update(provider, {}))
mock_ftp = ftp_connect.return_value.__enter__.return_value
self.assertEqual(mock_ftp.rename.call_count, len(FakeFTP.files))
for i, call in enumerate(mock_ftp.rename.call_args_list):
self.assertEqual(call[0], (FakeFTP.files[i][0], "error/{}".format(FakeFTP.files[i][0])))
def test_move_error_default(self, ftp_connect):
"""Check that error on ingestion use default path if "move_path_error" is empty string
feature requested in SDESK-1452
"""
provider = copy.deepcopy(PROVIDER)
provider['config']['move_path_error'] = ""
service = ftp.FTPFeedingService()
service._update(provider, {})
mock_ftp = ftp_connect.return_value.__enter__.return_value
self.assertEqual(mock_ftp.rename.call_count, len(FakeFTP.files))
for i, call in enumerate(mock_ftp.rename.call_args_list):
self.assertEqual(call[0], (FakeFTP.files[i][0], '{}/{}'.format(
ftp.DEFAULT_FAILURE_PATH, FakeFTP.files[i][0])))
def test_move_ingested(self, ftp_connect):
"""Check that ingested file is moved if "move" is set
feature requested in SDESK-468
"""
provider = copy.deepcopy(PROVIDER)
service = ftp.FTPFeedingService()
service._update(provider, {})
mock_ftp = ftp_connect.return_value.__enter__.return_value
self.assertEqual(mock_ftp.rename.call_count, len(FakeFTP.files))
for i, call in enumerate(mock_ftp.rename.call_args_list):
self.assertEqual(call[0],
(FakeFTP.files[i][0], 'dest_move/{}'.format(
FakeFTP.files[i][0])))
def test_move_backstop(self, ftp_connect):
"""Check that failing file is not moved if it's more recent thant INGEST_OLD_CONTENT_MINUTES"""
provider = copy.deepcopy(PROVIDER)
service = ftp.FTPFeedingService()
service._is_empty = mock.MagicMock(return_value=False)
ingest_items(service.update(provider, {}))
mock_ftp = ftp_connect.return_value.__enter__.return_value
# recent_file must not have been ingested
self.assertEqual(mock_ftp.rename.call_count, len(FakeFTPRecentFiles.files) - 1)
for i, call in enumerate(mock_ftp.rename.call_args_list):
self.assertNotEqual(
call[0][0],
"recent_file.xml",
)
def test_move_ingested_default(self, ftp_connect, *args):
"""Check that ingested file is moved to default path if "move" is empty string
feature requested in SDESK-1452
"""
provider = copy.deepcopy(PROVIDER)
provider['config']['ftp_move_path'] = ""
service = ftp.FTPFeedingService()
ingest_items(service.update(provider, {}))
mock_ftp = ftp_connect.return_value.__enter__.return_value
self.assertEqual(mock_ftp.rename.call_count, len(FakeFTP.files))
for i, call in enumerate(mock_ftp.rename.call_args_list):
self.assertEqual(call[0], (FakeFTP.files[i][0], '{}/{}'.format(
ftp.DEFAULT_SUCCESS_PATH, FakeFTP.files[i][0])))
def test_it_can_connect(self):
service = ftp.FTPFeedingService()
if 'FTP_URL' not in os.environ:
return
config = service.config_from_url(os.environ['FTP_URL'])
self.assertEqual('test', config['path'])
self.assertEqual('localhost', config['host'])
config['dest_path'] = tempfile.mkdtemp(prefix=PREFIX)
provider = {'config': config}
items = service._update(provider, {})
self.assertEqual(266, len(items))
provider['last_updated'] = utcnow()
self.assertEqual(0, len(service._update(provider, {})))
self.assertTrue(os.path.isdir(provider['config']['dest_path']))
self.assertEqual(266, len(os.listdir(provider['config']['dest_path'])))
def test_it_can_connect(self):
service = ftp.FTPFeedingService()
if "FTP_URL" not in os.environ:
return
config = service.config_from_url(os.environ["FTP_URL"])
self.assertEqual("test", config["path"])
self.assertEqual("localhost", config["host"])
config["dest_path"] = tempfile.mkdtemp(prefix=PREFIX)
provider = {"config": config}
items = service.update(provider, {})
self.assertEqual(266, len(items))
provider["last_updated"] = utcnow()
self.assertEqual(0, len(service.update(provider, {})))
self.assertTrue(os.path.isdir(provider["config"]["dest_path"]))
self.assertEqual(266, len(os.listdir(provider["config"]["dest_path"])))
def test_files_limit_no_move(self, *mocks):
"""Test file limits when move is off
feature requested in SDESK-3815
"""
update = {}
self.app.config["FTP_INGEST_FILES_LIST_LIMIT"] = 3
retrieve_and_parse, ftp_connect = mocks
provider = copy.deepcopy(PROVIDER)
provider["config"]["move"] = False
service = ftp.FTPFeedingService()
service._is_empty = mock.MagicMock(return_value=False)
mock_ftp = ftp_connect.return_value.__enter__.return_value
ingest_items(service.update(provider, update))
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 3)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164739", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 8)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164745", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 13)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164746", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 16)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164748", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 22)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164755", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 24)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164756", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
self.assertEqual(mock_ftp.rename.call_count, 0)
def test_files_limit_no_move(self, *mocks):
"""Test file limits when move is off
feature requested in SDESK-3815
"""
update = {}
self.app.config['FTP_INGEST_FILES_LIST_LIMIT'] = 3
retrieve_and_parse, ftp_connect = mocks
provider = copy.deepcopy(PROVIDER)
provider['config']['move'] = False
service = ftp.FTPFeedingService()
mock_ftp = ftp_connect.return_value.__enter__.return_value
service._update(provider, update)
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 3)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164739',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 8)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164745',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 13)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164746',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 16)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164748',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 22)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164755',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
self.assertEqual(retrieve_and_parse.call_count, 24)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164756',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
self.assertEqual(mock_ftp.rename.call_count, 0)
def test_files_limit_move(self, *mocks):
"""Test file limits when move is on
feature requested in SDESK-3815
"""
update = {}
self.app.config['FTP_INGEST_FILES_LIST_LIMIT'] = 3
retrieve_and_parse, ftp_connect = mocks
provider = copy.deepcopy(PROVIDER)
service = ftp.FTPFeedingService()
mock_ftp = ftp_connect.return_value.__enter__.return_value
service._update(provider, update)
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[3:])
self.assertEqual(retrieve_and_parse.call_count, 3)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164739',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[6:])
self.assertEqual(retrieve_and_parse.call_count, 6)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164745',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[9:])
self.assertEqual(retrieve_and_parse.call_count, 9)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164746',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[12:])
self.assertEqual(retrieve_and_parse.call_count, 12)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164748',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[15:])
self.assertEqual(retrieve_and_parse.call_count, 15)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164755',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
service._update(provider, update)
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[16:])
self.assertEqual(retrieve_and_parse.call_count, 16)
self.assertEqual(
provider['private']['last_processed_file_modify'],
datetime.datetime.strptime('20170517164756',
'%Y%m%d%H%M%S').replace(tzinfo=utc))
self.assertEqual(mock_ftp.rename.call_count, 16)
def test_files_limit_move(self, *mocks):
"""Test file limits when move is on
feature requested in SDESK-3815
"""
update = {}
self.app.config["FTP_INGEST_FILES_LIST_LIMIT"] = 3
retrieve_and_parse, ftp_connect = mocks
provider = copy.deepcopy(PROVIDER)
service = ftp.FTPFeedingService()
service._is_empty = mock.MagicMock(return_value=False)
mock_ftp = ftp_connect.return_value.__enter__.return_value
ingest_items(service.update(provider, update))
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[3:])
self.assertEqual(retrieve_and_parse.call_count, 3)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164739", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[6:])
self.assertEqual(retrieve_and_parse.call_count, 6)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164745", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[9:])
self.assertEqual(retrieve_and_parse.call_count, 9)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164746", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[12:])
self.assertEqual(retrieve_and_parse.call_count, 12)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164748", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[15:])
self.assertEqual(retrieve_and_parse.call_count, 15)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164755", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
ingest_items(service.update(provider, update))
provider.update(update)
# emulate moving files by reducing list
ftp_connect().__enter__().mlsd = mock.Mock()
ftp_connect().__enter__().mlsd.return_value = iter(FakeFTP.files[16:])
self.assertEqual(retrieve_and_parse.call_count, 16)
self.assertEqual(
provider["private"]["last_processed_file_modify"],
datetime.datetime.strptime("20170517164756", "%Y%m%d%H%M%S").replace(tzinfo=utc),
)
self.assertEqual(mock_ftp.rename.call_count, 16)
