def test_json(self):
"""Tests coverting a DeleteFiles message to and from JSON"""
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1)
file_2 = storage_test_utils.create_file(file_path=file_path_2)
# Add files to message
message = DeleteFiles()
message.purge = True
if message.can_fit_more():
message.add_file(file_1.id)
if message.can_fit_more():
message.add_file(file_2.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = DeleteFiles.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Both files should have been deleted
self.assertEqual(ScaleFile.objects.filter(id=file_1.id).count(), 0)
self.assertEqual(ScaleFile.objects.filter(id=file_2.id).count(), 0)
# No new messages
self.assertEqual(len(new_message.new_messages), 0)
def test_move_file_new_workspace_without_download(self, mock_message,
mock_delete, mock_upload,
mock_download,
mock_paths):
"""Tests moving a file"""
volume_path = os.path.join('the', 'volume', 'path')
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
full_path_file_1 = os.path.join(volume_path, file_path_1)
full_path_file_2 = os.path.join(volume_path, file_path_2)
file_1 = storage_test_utils.create_file(file_path=file_path_1,
workspace=self.old_workspace)
file_2 = storage_test_utils.create_file(file_path=file_path_2,
workspace=self.old_workspace)
file_ids = [file_1.id, file_2.id]
# Call function
move_files(file_ids,
new_workspace=self.new_workspace,
new_file_path=None)
# Check results
mock_download.assert_not_called()
mock_upload.assert_called()
mock_delete.assert_called()
def test_successfully(self, mock_execute, mock_exists):
"""Tests calling HostBroker.download_files() successfully"""
mock_exists.return_value = True
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
full_workspace_path_file_1 = os.path.join(volume_path,
workspace_path_file_1)
full_workspace_path_file_2 = os.path.join(volume_path,
workspace_path_file_2)
file_1 = storage_test_utils.create_file(
file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(
file_path=workspace_path_file_2)
file_1_dl = FileDownload(file_1, local_path_file_1, False)
file_2_dl = FileDownload(file_2, local_path_file_2, False)
# Call method to test
self.broker.download_files(volume_path, [file_1_dl, file_2_dl])
# Check results
two_calls = [
call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]),
call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])
]
mock_execute.assert_has_calls(two_calls)
def test_move_files(self, mock_conn_class):
"""Tests moving files successfully"""
s3_key_1 = MagicMock(Key)
s3_key_2 = MagicMock(Key)
mock_conn = MagicMock(BrokerConnection)
mock_conn.get_key.side_effect = [s3_key_1, s3_key_2]
mock_conn_class.return_value.__enter__ = Mock(return_value=mock_conn)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
file_1_mv = FileMove(file_1, new_workspace_path_1)
file_2_mv = FileMove(file_2, new_workspace_path_2)
# Call method to test
self.broker.move_files(None, [file_1_mv, file_2_mv])
# Check results
self.assertTrue(s3_key_1.copy.called)
self.assertTrue(s3_key_2.copy.called)
self.assertEqual(file_1.file_path, new_workspace_path_1)
self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_success(self):
"""Tests calling ScaleFileManager.move_files() successfully"""
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1)
new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path))
file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1)
new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path))
workspace_1.move_files = MagicMock()
workspace_2 = storage_test_utils.create_workspace()
file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2)
new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path))
file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2)
new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path))
workspace_2.move_files = MagicMock()
files = [FileMove(file_1, new_workspace_path_1), FileMove(file_2, new_workspace_path_2),
FileMove(file_3, new_workspace_path_3), FileMove(file_4, new_workspace_path_4)]
ScaleFile.objects.move_files(files)
workspace_1.move_files.assert_called_once_with([FileMove(file_1, new_workspace_path_1),
FileMove(file_2, new_workspace_path_2)])
workspace_2.move_files.assert_called_once_with([FileMove(file_3, new_workspace_path_3),
FileMove(file_4, new_workspace_path_4)])
def test_successfully(self, mock_copy, mock_chmod, mock_exists, mock_makedirs):
"""Tests calling HostBroker.upload_files() successfully"""
def new_exists(path):
return False
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1)
full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
file_1_up = FileUpload(file_1, local_path_file_1)
file_2_up = FileUpload(file_2, local_path_file_2)
# Call method to test
self.broker.upload_files(volume_path, [file_1_up, file_2_up])
# Check results
two_calls = [call(os.path.dirname(full_workspace_path_file_1), mode=0755),
call(os.path.dirname(full_workspace_path_file_2), mode=0755)]
mock_makedirs.assert_has_calls(two_calls)
two_calls = [call(local_path_file_1, full_workspace_path_file_1),
call(local_path_file_2, full_workspace_path_file_2)]
mock_copy.assert_has_calls(two_calls)
two_calls = [call(full_workspace_path_file_1, 0644), call(full_workspace_path_file_2, 0644)]
mock_chmod.assert_has_calls(two_calls)
def test_delete_file(self, mock_remove, mock_exists):
"""Tests removing a file"""
def new_exists(path):
return True
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
full_path_file_1 = os.path.join(volume_path, file_path_1)
full_path_file_2 = os.path.join(volume_path, file_path_2)
file_1 = storage_test_utils.create_file(file_path=file_path_1)
file_2 = storage_test_utils.create_file(file_path=file_path_2)
# Call function
test_1 = delete_files([file_1], volume_path, self.broker)
self.assertEqual(test_1, None)
test_2 = delete_files([file_2], volume_path, self.broker)
self.assertEqual(test_2, None)
# Check results
two_calls = [call(full_path_file_1), call(full_path_file_2)]
mock_remove.assert_has_calls(two_calls)
def test_successfully(self, mock_remove, mock_exists):
"""Tests calling HostBroker.delete_files() successfully"""
def new_exists(path):
return True
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
full_path_file_1 = os.path.join(volume_path, file_path_1)
full_path_file_2 = os.path.join(volume_path, file_path_2)
file_1 = storage_test_utils.create_file(file_path=file_path_1)
file_2 = storage_test_utils.create_file(file_path=file_path_2)
# Call method to test
self.broker.delete_files(volume_path, [file_1, file_2])
# Check results
two_calls = [call(full_path_file_1), call(full_path_file_2)]
mock_remove.assert_has_calls(two_calls)
self.assertTrue(file_1.is_deleted)
self.assertIsNotNone(file_1.deleted)
self.assertTrue(file_2.is_deleted)
self.assertIsNotNone(file_2.deleted)
def test_deleted_file(self):
'''Tests calling ScaleFileManager.download_files() with a deleted file'''
download_dir = os.path.join('download', 'dir')
work_dir = os.path.join('work', 'dir')
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
local_path_1 = u'my/local/path/file.txt'
file_2 = storage_test_utils.create_file(workspace=workspace_1)
local_path_2 = u'another/local/path/file.txt'
file_2.is_deleted = True
file_2.save()
file_3 = storage_test_utils.create_file(workspace=workspace_1)
local_path_3 = u'another/local/path/file.json'
workspace_1.download_files = MagicMock()
workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)
workspace_2 = storage_test_utils.create_workspace()
file_4 = storage_test_utils.create_file(workspace=workspace_2)
local_path_4 = u'my/local/path/4/file.txt'
file_5 = storage_test_utils.create_file(workspace=workspace_2)
local_path_5 = u'another/local/path/5/file.txt'
workspace_2.download_files = MagicMock()
workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)
files = [(file_1, local_path_1), (file_2, local_path_2), (file_3, local_path_3), (file_4, local_path_4),
(file_5, local_path_5)]
self.assertRaises(DeletedFile, ScaleFile.objects.download_files, download_dir, work_dir, files)
def test_inactive_workspace(self):
"""Tests calling ScaleFileManager.download_files() with an inactive workspace"""
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
local_path_1 = '/my/local/path/file.txt'
file_2 = storage_test_utils.create_file(workspace=workspace_1)
local_path_2 = '/another/local/path/file.txt'
file_3 = storage_test_utils.create_file(workspace=workspace_1)
local_path_3 = '/another/local/path/file.json'
workspace_1.download_files = MagicMock()
workspace_2 = storage_test_utils.create_workspace()
workspace_2.is_active = False
workspace_2.save()
file_4 = storage_test_utils.create_file(workspace=workspace_2)
local_path_4 = '/my/local/path/4/file.txt'
file_5 = storage_test_utils.create_file(workspace=workspace_2)
local_path_5 = '/another/local/path/5/file.txt'
files = [
FileDownload(file_1, local_path_1),
FileDownload(file_2, local_path_2),
FileDownload(file_3, local_path_3),
FileDownload(file_4, local_path_4),
FileDownload(file_5, local_path_5)
]
self.assertRaises(ArchivedWorkspace, ScaleFile.objects.download_files,
files)
def test_successfully(self, mock_execute, mock_exists):
"""Tests calling NfsBroker.download_files() successfully"""
def new_exists(path):
return False
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1)
full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
file_1_dl = FileDownload(file_1, local_path_file_1, False)
file_2_dl = FileDownload(file_2, local_path_file_2, False)
# Call method to test
self.broker.download_files(volume_path, [file_1_dl, file_2_dl])
# Check results
two_calls = [call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]),
call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])]
mock_execute.assert_has_calls(two_calls)
def test_create_message(self):
"""
Tests calling the create message function for DeleteFiles
"""
job = job_test_utils.create_job()
job_exe = job_test_utils.create_job_exe(job=job)
trigger = trigger_test_utils.create_trigger_event()
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file1.json')
file_path_3 = os.path.join('my_dir', 'my_file2.json')
file_path_4 = os.path.join('my_dir', 'my_file3.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1,
job_exe=job_exe)
file_2 = storage_test_utils.create_file(file_path=file_path_2,
job_exe=job_exe)
file_3 = storage_test_utils.create_file(file_path=file_path_3,
job_exe=job_exe)
file_4 = storage_test_utils.create_file(file_path=file_path_4,
job_exe=job_exe)
files = [file_1, file_2, file_3, file_4]
# No exception is success
create_delete_files_messages(files=files,
job_id=job.id,
trigger_id=trigger.id,
source_file_id=self.source_file.id,
purge=True)
def test_upload_files(self, mock_client_class):
"""Tests uploading files successfully"""
s3_object_1 = MagicMock()
s3_object_2 = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1, media_type='text/plain')
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2, media_type='application/json')
file_1_up = FileUpload(file_1, local_path_file_1)
file_2_up = FileUpload(file_2, local_path_file_2)
# Call method to test
mo = mock_open()
with patch('__builtin__.open', mo, create=True):
self.broker.upload_files(None, [file_1_up, file_2_up])
# Check results
self.assertTrue(s3_object_1.upload_file.called)
self.assertTrue(s3_object_2.upload_file.called)
self.assertEqual(s3_object_1.upload_file.call_args[0][1]['ContentType'], 'text/plain')
self.assertEqual(s3_object_2.upload_file.call_args[0][1]['ContentType'], 'application/json')
def test_no_files(self):
'''Tests calling ScaleFileManager.get_total_file_size() where no files match the file IDs'''
storage_test_utils.create_file(file_size=100)
file_size = ScaleFile.objects.get_total_file_size([4444444444, 555555555555, 666666666666])
self.assertEqual(file_size, 0)
def test_delete_files(self, mock_client_class):
"""Tests deleting files successfully"""
s3_object_1 = MagicMock()
s3_object_2 = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1)
file_2 = storage_test_utils.create_file(file_path=file_path_2)
# Call method to test
self.broker.delete_files(None, [file_1, file_2])
# Check results
self.assertTrue(s3_object_1.delete.called)
self.assertTrue(s3_object_2.delete.called)
self.assertTrue(file_1.is_deleted)
self.assertIsNotNone(file_1.deleted)
self.assertTrue(file_2.is_deleted)
self.assertIsNotNone(file_2.deleted)
def test_deleted_file(self):
'''Tests calling ScaleFileManager.download_files() with a deleted file'''
download_dir = os.path.join('download', 'dir')
work_dir = os.path.join('work', 'dir')
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
local_path_1 = u'my/local/path/file.txt'
file_2 = storage_test_utils.create_file(workspace=workspace_1)
local_path_2 = u'another/local/path/file.txt'
file_2.is_deleted = True
file_2.save()
file_3 = storage_test_utils.create_file(workspace=workspace_1)
local_path_3 = u'another/local/path/file.json'
workspace_1.download_files = MagicMock()
workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(
work_dir, workspace_1)
workspace_2 = storage_test_utils.create_workspace()
file_4 = storage_test_utils.create_file(workspace=workspace_2)
local_path_4 = u'my/local/path/4/file.txt'
file_5 = storage_test_utils.create_file(workspace=workspace_2)
local_path_5 = u'another/local/path/5/file.txt'
workspace_2.download_files = MagicMock()
workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(
work_dir, workspace_2)
files = [(file_1, local_path_1), (file_2, local_path_2),
(file_3, local_path_3), (file_4, local_path_4),
(file_5, local_path_5)]
self.assertRaises(DeletedFile, ScaleFile.objects.download_files,
download_dir, work_dir, files)
def test_download_files(self, mock_client_class):
"""Tests downloading files successfully"""
s3_object_1 = MagicMock()
s3_object_2 = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
file_1_dl = FileDownload(file_1, local_path_file_1)
file_2_dl = FileDownload(file_2, local_path_file_2)
# Call method to test
mo = mock_open()
with patch('__builtin__.open', mo, create=True):
self.broker.download_files(None, [file_1_dl, file_2_dl])
# Check results
self.assertTrue(s3_object_1.download_file.called)
self.assertTrue(s3_object_2.download_file.called)
def test_successfully(self, mock_remove, mock_exists):
"""Tests calling NfsBroker.delete_files() successfully"""
def new_exists(path):
return True
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
full_path_file_1 = os.path.join(volume_path, file_path_1)
full_path_file_2 = os.path.join(volume_path, file_path_2)
file_1 = storage_test_utils.create_file(file_path=file_path_1)
file_2 = storage_test_utils.create_file(file_path=file_path_2)
# Call method to test
self.broker.delete_files(volume_path, [file_1, file_2])
# Check results
two_calls = [call(full_path_file_1), call(full_path_file_2)]
mock_remove.assert_has_calls(two_calls)
self.assertTrue(file_1.is_deleted)
self.assertIsNotNone(file_1.deleted)
self.assertTrue(file_2.is_deleted)
self.assertIsNotNone(file_2.deleted)
def test_move_files(self, mock_client_class):
"""Tests moving files successfully"""
s3_object_1a = MagicMock()
s3_object_1b = MagicMock()
s3_object_2a = MagicMock()
s3_object_2b = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1a, s3_object_1b, s3_object_2a, s3_object_2b]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
file_1_mv = FileMove(file_1, new_workspace_path_1)
file_2_mv = FileMove(file_2, new_workspace_path_2)
# Call method to test
self.broker.move_files(None, [file_1_mv, file_2_mv])
# Check results
self.assertTrue(s3_object_1b.copy_from.called)
self.assertTrue(s3_object_1a.delete.called)
self.assertTrue(s3_object_2b.copy_from.called)
self.assertTrue(s3_object_2a.delete.called)
self.assertEqual(file_1.file_path, new_workspace_path_1)
self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_host_link_files(self, mock_execute, mock_client_class):
"""Tests sym-linking files successfully"""
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1)
full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
file_1_dl = FileDownload(file_1, local_path_file_1, True)
file_2_dl = FileDownload(file_2, local_path_file_2, True)
# Call method to test
self.broker.download_files(volume_path, [file_1_dl, file_2_dl])
# Check results
two_calls = [call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]),
call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])]
mock_execute.assert_has_calls(two_calls)
def test_host_link_files(self, mock_execute, mock_client_class):
"""Tests sym-linking files successfully"""
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1)
full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
file_1_dl = FileDownload(file_1, local_path_file_1, True)
file_2_dl = FileDownload(file_2, local_path_file_2, True)
# Call method to test
self.broker.download_files(volume_path, [file_1_dl, file_2_dl])
# Check results
two_calls = [call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]),
call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])]
mock_execute.assert_has_calls(two_calls)
def test_success(self):
"""Tests calling ScaleFileManager.download_files() successfully"""
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
local_path_1 = '/my/local/path/file.txt'
file_2 = storage_test_utils.create_file(workspace=workspace_1)
local_path_2 = '/another/local/path/file.txt'
file_3 = storage_test_utils.create_file(workspace=workspace_1)
local_path_3 = '/another/local/path/file.json'
workspace_1.setup_download_dir = MagicMock()
workspace_1.download_files = MagicMock()
workspace_2 = storage_test_utils.create_workspace()
file_4 = storage_test_utils.create_file(workspace=workspace_2)
local_path_4 = '/my/local/path/4/file.txt'
file_5 = storage_test_utils.create_file(workspace=workspace_2)
local_path_5 = '/another/local/path/5/file.txt'
workspace_2.setup_download_dir = MagicMock()
workspace_2.download_files = MagicMock()
files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False),
FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False),
FileDownload(file_5, local_path_5, False)]
ScaleFile.objects.download_files(files)
workspace_1.download_files.assert_called_once_with([FileDownload(file_1, local_path_1, False),
FileDownload(file_2, local_path_2, False),
FileDownload(file_3, local_path_3, False)])
workspace_2.download_files.assert_called_once_with([FileDownload(file_4, local_path_4, False),
FileDownload(file_5, local_path_5, False)])
def test_deleted_file(self):
'''Tests calling ScaleFileManager.move_files() with a deleted file'''
work_dir = os.path.join('work', 'dir')
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1)
new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path))
file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1)
file_2.is_deleted = True
file_2.save()
new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path))
workspace_1.move_files = MagicMock()
workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)
workspace_2 = storage_test_utils.create_workspace()
workspace_2.is_active = False
workspace_2.save()
file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2)
new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path))
file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2)
new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path))
workspace_2.move_files = MagicMock()
workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)
files = [(file_1, new_workspace_path_1), (file_2, new_workspace_path_2), (file_3, new_workspace_path_3),
(file_4, new_workspace_path_4)]
self.assertRaises(DeletedFile, ScaleFile.objects.move_files, work_dir, files)
def test_deleted_file(self):
"""Tests calling ScaleFileManager.download_files() with a deleted file"""
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
local_path_1 = '/my/local/path/file.txt'
file_2 = storage_test_utils.create_file(workspace=workspace_1)
local_path_2 = '/another/local/path/file.txt'
file_2.is_deleted = True
file_2.save()
file_3 = storage_test_utils.create_file(workspace=workspace_1)
local_path_3 = '/another/local/path/file.json'
workspace_1.download_files = MagicMock()
workspace_2 = storage_test_utils.create_workspace()
file_4 = storage_test_utils.create_file(workspace=workspace_2)
local_path_4 = '/my/local/path/4/file.txt'
file_5 = storage_test_utils.create_file(workspace=workspace_2)
local_path_5 = '/another/local/path/5/file.txt'
workspace_2.download_files = MagicMock()
files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False),
FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False),
FileDownload(file_5, local_path_5, False)]
self.assertRaises(DeletedFile, ScaleFile.objects.download_files, files)
def test_move_files(self, mock_client_class):
"""Tests moving files successfully"""
s3_object_1a = MagicMock()
s3_object_1b = MagicMock()
s3_object_2a = MagicMock()
s3_object_2b = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1a, s3_object_1b, s3_object_2a, s3_object_2b]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
file_1_mv = FileMove(file_1, new_workspace_path_1)
file_2_mv = FileMove(file_2, new_workspace_path_2)
# Call method to test
self.broker.move_files(None, [file_1_mv, file_2_mv])
# Check results
self.assertTrue(s3_object_1b.copy_from.called)
self.assertTrue(s3_object_1a.delete.called)
self.assertTrue(s3_object_2b.copy_from.called)
self.assertTrue(s3_object_2a.delete.called)
self.assertEqual(file_1.file_path, new_workspace_path_1)
self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_inactive_workspace(self):
"""Tests calling ScaleFileManager.move_files() with an inactive workspace"""
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(file_name='my_file_1.txt',
workspace=workspace_1)
new_workspace_path_1 = os.path.join('my', 'new', 'path', '1',
os.path.basename(file_1.file_path))
file_2 = storage_test_utils.create_file(file_name='my_file_2.txt',
workspace=workspace_1)
new_workspace_path_2 = os.path.join('my', 'new', 'path', '2',
os.path.basename(file_2.file_path))
workspace_1.move_files = MagicMock()
workspace_2 = storage_test_utils.create_workspace()
workspace_2.is_active = False
workspace_2.save()
file_3 = storage_test_utils.create_file(file_name='my_file_3.txt',
workspace=workspace_2)
new_workspace_path_3 = os.path.join('my', 'new', 'path', '3',
os.path.basename(file_3.file_path))
file_4 = storage_test_utils.create_file(file_name='my_file_4.txt',
workspace=workspace_2)
new_workspace_path_4 = os.path.join('my', 'new', 'path', '4',
os.path.basename(file_4.file_path))
workspace_2.move_files = MagicMock()
files = [
FileMove(file_1, new_workspace_path_1),
FileMove(file_2, new_workspace_path_2),
FileMove(file_3, new_workspace_path_3),
FileMove(file_4, new_workspace_path_4)
]
self.assertRaises(ArchivedWorkspace, ScaleFile.objects.move_files,
files)
def test_move_files(self, mock_conn_class):
"""Tests moving files successfully"""
s3_key_1 = MagicMock(Key)
s3_key_2 = MagicMock(Key)
mock_conn = MagicMock(BrokerConnection)
mock_conn.get_key.side_effect = [s3_key_1, s3_key_2]
mock_conn_class.return_value.__enter__ = Mock(return_value=mock_conn)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
file_1_mv = FileMove(file_1, new_workspace_path_1)
file_2_mv = FileMove(file_2, new_workspace_path_2)
# Call method to test
self.broker.move_files(None, [file_1_mv, file_2_mv])
# Check results
self.assertTrue(s3_key_1.copy.called)
self.assertTrue(s3_key_2.copy.called)
self.assertEqual(file_1.file_path, new_workspace_path_1)
self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_download_files(self, mock_client_class):
"""Tests downloading files successfully"""
s3_object_1 = MagicMock()
s3_object_2 = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1)
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2)
file_1_dl = FileDownload(file_1, local_path_file_1, False)
file_2_dl = FileDownload(file_2, local_path_file_2, False)
# Call method to test
mo = mock_open()
with patch('__builtin__.open', mo, create=True):
self.broker.download_files(None, [file_1_dl, file_2_dl])
# Check results
self.assertTrue(s3_object_1.download_file.called)
self.assertTrue(s3_object_2.download_file.called)
def test_delete_files(self, mock_client_class):
"""Tests deleting files successfully"""
s3_object_1 = MagicMock()
s3_object_2 = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1)
file_2 = storage_test_utils.create_file(file_path=file_path_2)
# Call method to test
self.broker.delete_files(None, [file_1, file_2])
# Check results
self.assertTrue(s3_object_1.delete.called)
self.assertTrue(s3_object_2.delete.called)
self.assertTrue(file_1.is_deleted)
self.assertIsNotNone(file_1.deleted)
self.assertTrue(file_2.is_deleted)
self.assertIsNotNone(file_2.deleted)
def test_upload_files(self, mock_client_class):
"""Tests uploading files successfully"""
s3_object_1 = MagicMock()
s3_object_2 = MagicMock()
mock_client = MagicMock(S3Client)
mock_client.get_object.side_effect = [s3_object_1, s3_object_2]
mock_client_class.return_value.__enter__ = Mock(return_value=mock_client)
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
local_path_file_1 = os.path.join('my_dir_1', file_name_1)
local_path_file_2 = os.path.join('my_dir_2', file_name_2)
workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1)
workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2)
file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1, media_type='text/plain')
file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2, media_type='application/json')
file_1_up = FileUpload(file_1, local_path_file_1)
file_2_up = FileUpload(file_2, local_path_file_2)
# Call method to test
mo = mock_open()
with patch('__builtin__.open', mo, create=True):
self.broker.upload_files(None, [file_1_up, file_2_up])
# Check results
self.assertTrue(s3_object_1.upload_file.called)
self.assertTrue(s3_object_2.upload_file.called)
self.assertEqual(s3_object_1.upload_file.call_args[0][1]['ContentType'], 'text/plain')
self.assertEqual(s3_object_2.upload_file.call_args[0][1]['ContentType'], 'application/json')
def setUp(self):
django.setup()
self.file_1 = storage_utils.create_file(file_name='my_json_file.json',
media_type='application/json')
self.file_2 = storage_utils.create_file(file_name='my_text_file_1.txt',
media_type='text/plain')
self.file_3 = storage_utils.create_file(file_name='my_text_file_2.txt',
media_type='text/plain')
def test_no_files(self):
'''Tests calling ScaleFileManager.get_total_file_size() where no files match the file IDs'''
storage_test_utils.create_file(file_size=100)
file_size = ScaleFile.objects.get_total_file_size(
[4444444444, 555555555555, 666666666666])
self.assertEqual(file_size, 0)
def test_success(self):
'''Tests calling ScaleFileManager.get_total_file_size() successfully'''
file_1 = storage_test_utils.create_file(file_size=100)
file_2 = storage_test_utils.create_file(file_size=300)
storage_test_utils.create_file(file_size=700)
file_size = ScaleFile.objects.get_total_file_size([file_1.id, file_2.id])
self.assertEqual(file_size, 400)
def setUp(self):
django.setup()
self.job_1 = job_test_utils.create_job()
self.trigger_1 = trigger_test_utils.create_trigger_event()
self.job_exe_1 = job_test_utils.create_job_exe(job=self.job_1)
self.file_1 = storage_test_utils.create_file(job_exe=self.job_exe_1)
self.source_file = storage_test_utils.create_file(file_type='SOURCE')
self.workspace = storage_test_utils.create_workspace()
def test_success(self, mock_makedirs):
'''Tests calling ScaleFileManager.move_files() successfully'''
work_dir = os.path.join('work', 'dir')
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(file_name='my_file_1.txt',
workspace=workspace_1)
old_path_1 = file_1.file_path
new_workspace_path_1 = os.path.join('my', 'new', 'path', '1',
os.path.basename(file_1.file_path))
file_2 = storage_test_utils.create_file(file_name='my_file_2.txt',
workspace=workspace_1)
old_path_2 = file_2.file_path
new_workspace_path_2 = os.path.join('my', 'new', 'path', '2',
os.path.basename(file_2.file_path))
workspace_1.move_files = MagicMock()
workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(
work_dir, workspace_1)
workspace_2 = storage_test_utils.create_workspace()
file_3 = storage_test_utils.create_file(file_name='my_file_3.txt',
workspace=workspace_2)
old_path_3 = file_3.file_path
new_workspace_path_3 = os.path.join('my', 'new', 'path', '3',
os.path.basename(file_3.file_path))
file_4 = storage_test_utils.create_file(file_name='my_file_4.txt',
workspace=workspace_2)
old_path_4 = file_4.file_path
new_workspace_path_4 = os.path.join('my', 'new', 'path', '4',
os.path.basename(file_4.file_path))
workspace_2.move_files = MagicMock()
workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(
work_dir, workspace_2)
files = [
(file_1, new_workspace_path_1), (file_2, new_workspace_path_2),
(file_3, new_workspace_path_3), (file_4, new_workspace_path_4)
]
ScaleFile.objects.move_files(work_dir, files)
workspace_1.move_files.assert_called_once_with(
workspace_1_work_dir, [(old_path_1, new_workspace_path_1),
(old_path_2, new_workspace_path_2)])
workspace_2.move_files.assert_called_once_with(
workspace_2_work_dir, [(old_path_3, new_workspace_path_3),
(old_path_4, new_workspace_path_4)])
# Check models for new workspace paths
new_file_1 = ScaleFile.objects.get(id=file_1.id)
self.assertEqual(new_file_1.file_path, new_workspace_path_1)
new_file_2 = ScaleFile.objects.get(id=file_2.id)
self.assertEqual(new_file_2.file_path, new_workspace_path_2)
new_file_3 = ScaleFile.objects.get(id=file_3.id)
self.assertEqual(new_file_3.file_path, new_workspace_path_3)
new_file_4 = ScaleFile.objects.get(id=file_4.id)
self.assertEqual(new_file_4.file_path, new_workspace_path_4)
def test_success(self):
'''Tests calling ScaleFileManager.get_total_file_size() successfully'''
file_1 = storage_test_utils.create_file(file_size=100)
file_2 = storage_test_utils.create_file(file_size=300)
storage_test_utils.create_file(file_size=700)
file_size = ScaleFile.objects.get_total_file_size(
[file_1.id, file_2.id])
self.assertEqual(file_size, 400)
def test_execute_force_stop(self):
"""Tests calling DeleteFile.execute() successfully"""
# Create PurgeResults entry
source_file = storage_test_utils.create_file()
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger,
force_stop_purge=True)
self.assertEqual(
PurgeResults.objects.values_list(
'num_products_deleted',
flat=True).get(source_file_id=source_file.id), 0)
job = job_test_utils.create_job()
job_exe = job_test_utils.create_job_exe(job=job)
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file1.json')
file_path_3 = os.path.join('my_dir', 'my_file2.json')
file_path_4 = os.path.join('my_dir', 'my_file3.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1,
job_exe=job_exe)
file_2 = storage_test_utils.create_file(file_path=file_path_2,
job_exe=job_exe)
file_3 = storage_test_utils.create_file(file_path=file_path_3,
job_exe=job_exe)
file_4 = storage_test_utils.create_file(file_path=file_path_4,
job_exe=job_exe)
# Add files to message
message = DeleteFiles()
message.purge = True
message.job_id = job.id
message.source_file_id = source_file.id
message.trigger_id = trigger.id
if message.can_fit_more():
message.add_file(file_1.id)
if message.can_fit_more():
message.add_file(file_2.id)
if message.can_fit_more():
message.add_file(file_3.id)
if message.can_fit_more():
message.add_file(file_4.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Check results are accurate
self.assertEqual(
PurgeResults.objects.values_list(
'num_products_deleted',
flat=True).get(source_file_id=source_file.id), 0)
def test_schedule_date_range_data_ended(self):
"""Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction"""
file1 = storage_test_utils.create_file()
file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc)
file1.data_ended = datetime.datetime(2016, 1, 10, tzinfo=utc)
file1.save()
data1 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file1.id,
}],
'workspace_id': self.workspace.id,
}
recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data1),
event=self.event).recipe
file2 = storage_test_utils.create_file()
file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc)
file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc)
file2.save()
data2 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file2.id,
}],
'workspace_id': self.workspace.id,
}
Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data2),
event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'type': 'data',
'ended': '2016-01-15T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 1)
self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def test_inactive_workspace(self):
"""Tests calling deleting files from an inactive workspace"""
workspace_1 = storage_test_utils.create_workspace()
workspace_1.download_files = MagicMock()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
workspace_2 = storage_test_utils.create_workspace(is_active=False)
file_2 = storage_test_utils.create_file(workspace=workspace_2)
files = [file_1, file_2]
self.assertRaises(ArchivedWorkspace, ScaleFile.objects.delete_files, files)
def test_successfully(self, mock_move, mock_chmod, mock_exists,
mock_makedirs):
"""Tests calling HostBroker.move_files() successfully"""
def new_exists(path):
return path.count('new') == 0
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)
full_old_workspace_path_1 = os.path.join(volume_path,
old_workspace_path_1)
full_old_workspace_path_2 = os.path.join(volume_path,
old_workspace_path_2)
full_new_workspace_path_1 = os.path.join(volume_path,
new_workspace_path_1)
full_new_workspace_path_2 = os.path.join(volume_path,
new_workspace_path_2)
file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
file_1_mv = FileMove(file_1, new_workspace_path_1)
file_2_mv = FileMove(file_2, new_workspace_path_2)
# Call method to test
self.broker.move_files(volume_path, [file_1_mv, file_2_mv])
# Check results
two_calls = [
call(os.path.dirname(full_new_workspace_path_1), mode=0755),
call(os.path.dirname(full_new_workspace_path_2), mode=0755)
]
mock_makedirs.assert_has_calls(two_calls)
two_calls = [
call(full_old_workspace_path_1, full_new_workspace_path_1),
call(full_old_workspace_path_2, full_new_workspace_path_2)
]
mock_move.assert_has_calls(two_calls)
two_calls = [
call(full_new_workspace_path_1, 0644),
call(full_new_workspace_path_2, 0644)
]
mock_chmod.assert_has_calls(two_calls)
self.assertEqual(file_1.file_path, new_workspace_path_1)
self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_schedule_date_range_data_ended(self):
"""Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction"""
file1 = storage_test_utils.create_file()
file1.data_started = datetime.datetime(2016, 1, 1)
file1.data_ended = datetime.datetime(2016, 1, 10)
file1.save()
data1 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file1.id,
}],
'workspace_id': self.workspace.id,
}
recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data1),
event=self.event).recipe
file2 = storage_test_utils.create_file()
file2.data_started = datetime.datetime(2016, 2, 1)
file2.data_ended = datetime.datetime(2016, 2, 10)
file2.save()
data2 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file2.id,
}],
'workspace_id': self.workspace.id,
}
Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data2), event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'type': 'data',
'ended': '2016-01-15T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 1)
self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def setUp(self):
django.setup()
self.workspace = storage_utils.create_workspace()
self.file_name_1 = 'my_file.txt'
self.media_type_1 = 'text/plain'
self.product_file_1 = storage_utils.create_file(file_name=self.file_name_1, media_type=self.media_type_1,
workspace=self.workspace)
self.file_name_2 = 'my_file.txt'
self.media_type_2 = 'text/plain'
self.product_file_2 = storage_utils.create_file(file_name=self.file_name_2, media_type=self.media_type_2,
workspace=self.workspace)
self.invalid_product_file_id = long(999)
def setUp(self):
django.setup()
self.workspace = storage_utils.create_workspace()
self.file_name_1 = 'my_file.txt'
self.media_type_1 = 'text/plain'
self.product_file_1 = storage_utils.create_file(file_name=self.file_name_1, media_type=self.media_type_1,
workspace=self.workspace)
self.file_name_2 = 'my_file.txt'
self.media_type_2 = 'text/plain'
self.product_file_2 = storage_utils.create_file(file_name=self.file_name_2, media_type=self.media_type_2,
workspace=self.workspace)
self.invalid_product_file_id = long(999)
def setUp(self):
django.setup()
self.count = 1
self.job_type = job_test_utils.create_seed_job_type()
self.job = job_test_utils.create_job(job_type=self.job_type)
self.job_exe = job_test_utils.create_job_exe(status='COMPLETED', job=self.job)
self.wp1 = storage_test_utils.create_workspace()
self.wp2 = storage_test_utils.create_workspace()
self.prod1 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp1, job_exe=self.job_exe)
self.prod2 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp1, job_exe=self.job_exe)
self.prod3 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp2, job_exe=self.job_exe)
self.file_1 = storage_test_utils.create_file(file_type='SOURCE')
self.event = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=self.file_1.id, trigger_event=self.event)
def test_success(self):
"""Tests deleting files successfully"""
workspace_1 = storage_test_utils.create_workspace()
workspace_1.delete_files = MagicMock()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
workspace_2 = storage_test_utils.create_workspace()
workspace_2.delete_files = MagicMock()
file_2 = storage_test_utils.create_file(workspace=workspace_2)
files = [file_1, file_2]
ScaleFile.objects.delete_files(files)
workspace_1.delete_files.assert_called_once_with([file_1])
workspace_2.delete_files.assert_called_once_with([file_2])
def test_url_file_slash(self):
'''Tests building a URL for a file where the file path URL has a leading slash.'''
ws = storage_test_utils.create_workspace(name='test', base_url='http://localhost')
file = storage_test_utils.create_file(file_name='test.txt', file_path='/file/path/test.txt',
workspace=ws)
self.assertEqual(file.url, 'http://localhost/file/path/test.txt')
def test_inputs_and_products(self):
"""Tests creating links for inputs and then later replacing with generated products."""
file_8 = storage_test_utils.create_file()
parent_ids = [self.file_4.id, self.file_6.id, self.file_7.id]
child_ids = [file_8.id]
job_exe = job_test_utils.create_job_exe()
recipe_test_utils.create_recipe_job(job=job_exe.job)
# First create only the input files
FileAncestryLink.objects.create_file_ancestry_links(parent_ids, None, job_exe)
# Replace the inputs with the new links for both inputs and products
FileAncestryLink.objects.create_file_ancestry_links(parent_ids, child_ids, job_exe)
# Make sure the old entries were deleted
old_direct_qry = FileAncestryLink.objects.filter(descendant__isnull=True, job_exe=job_exe,
ancestor_job__isnull=True)
self.assertEqual(len(old_direct_qry), 0)
old_indirect_qry = FileAncestryLink.objects.filter(descendant__isnull=True, job_exe=job_exe,
ancestor_job__isnull=False)
self.assertEqual(len(old_indirect_qry), 0)
direct_qry = FileAncestryLink.objects.filter(descendant=file_8, job_exe=job_exe, ancestor_job__isnull=True)
self.assertEqual(direct_qry.count(), 3)
file_8_parent_ids = {link.ancestor_id for link in direct_qry}
self.assertSetEqual(file_8_parent_ids, {self.file_4.id, self.file_6.id, self.file_7.id})
indirect_qry = FileAncestryLink.objects.filter(descendant=file_8, job_exe=job_exe, ancestor_job__isnull=False)
self.assertEqual(indirect_qry.count(), 3)
file_8_ancestor_ids = {link.ancestor_id for link in indirect_qry}
self.assertSetEqual(file_8_ancestor_ids, {self.file_1.id, self.file_2.id, self.file_3.id})
def test_deleted_file(self):
"""Tests attempting to delete a file that is already deleted"""
workspace_1 = storage_test_utils.create_workspace()
workspace_1.delete_files = MagicMock()
file_1a = storage_test_utils.create_file(workspace=workspace_1)
file_1b = storage_test_utils.create_file(workspace=workspace_1, is_deleted=True)
workspace_2 = storage_test_utils.create_workspace()
workspace_2.delete_files = MagicMock()
file_2 = storage_test_utils.create_file(workspace=workspace_2)
files = [file_1a, file_1b, file_2]
ScaleFile.objects.delete_files(files)
workspace_1.delete_files.assert_called_once_with([file_1a, file_1b])
workspace_2.delete_files.assert_called_once_with([file_2])
def setUp(self):
django.setup()
# Generation 1
self.file_1 = storage_test_utils.create_file()
self.file_2 = storage_test_utils.create_file()
# Generation 2
job_exe_1 = job_test_utils.create_job_exe()
recipe_job_1 = recipe_test_utils.create_recipe_job(job=job_exe_1.job)
self.file_3 = prod_test_utils.create_product(job_exe=job_exe_1)
self.file_4 = prod_test_utils.create_product(job_exe=job_exe_1)
self.file_5 = prod_test_utils.create_product(job_exe=job_exe_1)
# Generation 3
job_exe_2 = job_test_utils.create_job_exe()
recipe_job_2 = recipe_test_utils.create_recipe_job(job=job_exe_2.job)
self.file_6 = prod_test_utils.create_product(job_exe=job_exe_2)
# Stand alone file
self.file_7 = prod_test_utils.create_product()
# First job links generation 1 to 2
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_3, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_4, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_5, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_3, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_4, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_5, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
# Second job links generation 2 to 3
FileAncestryLink.objects.create(ancestor=self.file_3, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe,
ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe,
ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
def test_set_deleted(self):
"""Tests marking a file as deleted."""
scale_file = storage_test_utils.create_file()
scale_file.set_deleted()
self.assertTrue(scale_file.is_deleted)
self.assertIsNotNone(scale_file.deleted)
def test_success(self):
'''Tests calling ScaleFileManager.move_files() successfully'''
work_dir = os.path.join('work', 'dir')
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1)
old_path_1 = file_1.file_path
new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path))
file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1)
old_path_2 = file_2.file_path
new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path))
workspace_1.move_files = MagicMock()
workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)
workspace_2 = storage_test_utils.create_workspace()
file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2)
old_path_3 = file_3.file_path
new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path))
file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2)
old_path_4 = file_4.file_path
new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path))
workspace_2.move_files = MagicMock()
workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)
files = [(file_1, new_workspace_path_1), (file_2, new_workspace_path_2), (file_3, new_workspace_path_3),
(file_4, new_workspace_path_4)]
ScaleFile.objects.move_files(work_dir, files)
workspace_1.move_files.assert_called_once_with(workspace_1_work_dir,
[(old_path_1, new_workspace_path_1),
(old_path_2, new_workspace_path_2)])
workspace_2.move_files.assert_called_once_with(workspace_2_work_dir,
[(old_path_3, new_workspace_path_3),
(old_path_4, new_workspace_path_4)])
# Check models for new workspace paths
new_file_1 = ScaleFile.objects.get(id=file_1.id)
self.assertEqual(new_file_1.file_path, new_workspace_path_1)
new_file_2 = ScaleFile.objects.get(id=file_2.id)
self.assertEqual(new_file_2.file_path, new_workspace_path_2)
new_file_3 = ScaleFile.objects.get(id=file_3.id)
self.assertEqual(new_file_3.file_path, new_workspace_path_3)
new_file_4 = ScaleFile.objects.get(id=file_4.id)
self.assertEqual(new_file_4.file_path, new_workspace_path_4)
def setUp(self):
django.setup()
self.input_name_1 = 'Test Input 1'
self.output_name_1 = 'Test Output 1'
interface_1 = {
'version': '1.0',
'command': 'my_cmd',
'command_arguments': 'args',
'input_data': [{
'name': self.input_name_1,
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': self.output_name_1,
'type': 'files',
'media_type': 'image/png',
}],
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
self.input_name_2 = 'Test Input 2'
self.output_name_2 = 'Test Output 2'
interface_2 = {
'version': '1.0',
'command': 'my_cmd',
'command_arguments': 'args',
'input_data': [{
'name': self.input_name_2,
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': self.output_name_2,
'type': 'file',
}],
}
self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)
self.input_name_3 = 'Test Input 3'
self.output_name_3 = 'Test Output 3'
interface_3 = {
'version': '1.0',
'command': 'my_cmd',
'command_arguments': 'args',
'input_data': [{
'name': self.input_name_3,
'type': 'file',
'media_types': ['text/plain'],
}],
}
self.job_type_3 = job_test_utils.create_job_type(interface=interface_3)
self.file_1 = storage_test_utils.create_file(media_type='text/plain')
def test_successfully(self, mock_move, mock_chmod, mock_exists, mock_makedirs):
"""Tests calling NfsBroker.move_files() successfully"""
def new_exists(path):
return False
mock_exists.side_effect = new_exists
volume_path = os.path.join('the', 'volume', 'path')
file_name_1 = 'my_file.txt'
file_name_2 = 'my_file.json'
old_workspace_path_1 = os.path.join('my_dir_1', file_name_1)
old_workspace_path_2 = os.path.join('my_dir_2', file_name_2)
new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1)
new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2)
full_old_workspace_path_1 = os.path.join(volume_path, old_workspace_path_1)
full_old_workspace_path_2 = os.path.join(volume_path, old_workspace_path_2)
full_new_workspace_path_1 = os.path.join(volume_path, new_workspace_path_1)
full_new_workspace_path_2 = os.path.join(volume_path, new_workspace_path_2)
file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1)
file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2)
file_1_mv = FileMove(file_1, new_workspace_path_1)
file_2_mv = FileMove(file_2, new_workspace_path_2)
# Call method to test
self.broker.move_files(volume_path, [file_1_mv, file_2_mv])
# Check results
two_calls = [call(os.path.dirname(full_new_workspace_path_1), mode=0755),
call(os.path.dirname(full_new_workspace_path_2), mode=0755)]
mock_makedirs.assert_has_calls(two_calls)
two_calls = [call(full_old_workspace_path_1, full_new_workspace_path_1),
call(full_old_workspace_path_2, full_new_workspace_path_2)]
mock_move.assert_has_calls(two_calls)
two_calls = [call(full_new_workspace_path_1, 0644), call(full_new_workspace_path_2, 0644)]
mock_chmod.assert_has_calls(two_calls)
self.assertEqual(file_1.file_path, new_workspace_path_1)
self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_inactive_workspace(self):
"""Tests calling ScaleFileManager.move_files() with an inactive workspace"""
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1)
new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path))
file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1)
new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path))
workspace_1.move_files = MagicMock()
workspace_2 = storage_test_utils.create_workspace()
workspace_2.is_active = False
workspace_2.save()
file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2)
new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path))
file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2)
new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path))
workspace_2.move_files = MagicMock()
files = [FileMove(file_1, new_workspace_path_1), FileMove(file_2, new_workspace_path_2),
FileMove(file_3, new_workspace_path_3), FileMove(file_4, new_workspace_path_4)]
self.assertRaises(ArchivedWorkspace, ScaleFile.objects.move_files, files)
def test_success(self):
'''Tests calling ScaleFileManager.download_files() successfully'''
download_dir = os.path.join('download', 'dir')
work_dir = os.path.join('work', 'dir')
workspace_1 = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace_1)
local_path_1 = u'my/local/path/file.txt'
file_2 = storage_test_utils.create_file(workspace=workspace_1)
local_path_2 = u'another/local/path/file.txt'
file_3 = storage_test_utils.create_file(workspace=workspace_1)
local_path_3 = u'another/local/path/file.json'
workspace_1.setup_download_dir = MagicMock()
workspace_1.download_files = MagicMock()
workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1)
workspace_2 = storage_test_utils.create_workspace()
file_4 = storage_test_utils.create_file(workspace=workspace_2)
local_path_4 = u'my/local/path/4/file.txt'
file_5 = storage_test_utils.create_file(workspace=workspace_2)
local_path_5 = u'another/local/path/5/file.txt'
workspace_2.setup_download_dir = MagicMock()
workspace_2.download_files = MagicMock()
workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2)
files = [(file_1, local_path_1), (file_2, local_path_2), (file_3, local_path_3), (file_4, local_path_4),
(file_5, local_path_5)]
ScaleFile.objects.download_files(download_dir, work_dir, files)
workspace_1.setup_download_dir.assert_called_once_with(download_dir, workspace_1_work_dir)
workspace_1.download_files.assert_called_once_with(download_dir, workspace_1_work_dir,
[(file_1.file_path, local_path_1),
(file_2.file_path, local_path_2),
(file_3.file_path, local_path_3)])
workspace_2.setup_download_dir.assert_called_once_with(download_dir, workspace_2_work_dir)
workspace_2.download_files.assert_called_once_with(download_dir, workspace_2_work_dir,
[(file_4.file_path, local_path_4),
(file_5.file_path, local_path_5)])
