def test_json(self): """Tests coverting a DeleteFiles message to and from JSON""" file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file.json') file_1 = storage_test_utils.create_file(file_path=file_path_1) file_2 = storage_test_utils.create_file(file_path=file_path_2) # Add files to message message = DeleteFiles() message.purge = True if message.can_fit_more(): message.add_file(file_1.id) if message.can_fit_more(): message.add_file(file_2.id) # Convert message to JSON and back, and then execute message_json_dict = message.to_json() new_message = DeleteFiles.from_json(message_json_dict) result = new_message.execute() self.assertTrue(result) # Both files should have been deleted self.assertEqual(ScaleFile.objects.filter(id=file_1.id).count(), 0) self.assertEqual(ScaleFile.objects.filter(id=file_2.id).count(), 0) # No new messages self.assertEqual(len(new_message.new_messages), 0)
def test_move_file_new_workspace_without_download(self, mock_message, mock_delete, mock_upload, mock_download, mock_paths): """Tests moving a file""" volume_path = os.path.join('the', 'volume', 'path') file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file.json') full_path_file_1 = os.path.join(volume_path, file_path_1) full_path_file_2 = os.path.join(volume_path, file_path_2) file_1 = storage_test_utils.create_file(file_path=file_path_1, workspace=self.old_workspace) file_2 = storage_test_utils.create_file(file_path=file_path_2, workspace=self.old_workspace) file_ids = [file_1.id, file_2.id] # Call function move_files(file_ids, new_workspace=self.new_workspace, new_file_path=None) # Check results mock_download.assert_not_called() mock_upload.assert_called() mock_delete.assert_called()
def test_successfully(self, mock_execute, mock_exists): """Tests calling HostBroker.download_files() successfully""" mock_exists.return_value = True volume_path = os.path.join('the', 'volume', 'path') file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1) full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2) file_1 = storage_test_utils.create_file( file_path=workspace_path_file_1) file_2 = storage_test_utils.create_file( file_path=workspace_path_file_2) file_1_dl = FileDownload(file_1, local_path_file_1, False) file_2_dl = FileDownload(file_2, local_path_file_2, False) # Call method to test self.broker.download_files(volume_path, [file_1_dl, file_2_dl]) # Check results two_calls = [ call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]), call(['ln', '-s', full_workspace_path_file_2, local_path_file_2]) ] mock_execute.assert_has_calls(two_calls)
def test_move_files(self, mock_conn_class): """Tests moving files successfully""" s3_key_1 = MagicMock(Key) s3_key_2 = MagicMock(Key) mock_conn = MagicMock(BrokerConnection) mock_conn.get_key.side_effect = [s3_key_1, s3_key_2] mock_conn_class.return_value.__enter__ = Mock(return_value=mock_conn) file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' old_workspace_path_1 = os.path.join('my_dir_1', file_name_1) old_workspace_path_2 = os.path.join('my_dir_2', file_name_2) new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1) new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2) file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1) file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2) file_1_mv = FileMove(file_1, new_workspace_path_1) file_2_mv = FileMove(file_2, new_workspace_path_2) # Call method to test self.broker.move_files(None, [file_1_mv, file_2_mv]) # Check results self.assertTrue(s3_key_1.copy.called) self.assertTrue(s3_key_2.copy.called) self.assertEqual(file_1.file_path, new_workspace_path_1) self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_success(self): """Tests calling ScaleFileManager.move_files() successfully""" workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1) new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path)) file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1) new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path)) workspace_1.move_files = MagicMock() workspace_2 = storage_test_utils.create_workspace() file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2) new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path)) file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2) new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path)) workspace_2.move_files = MagicMock() files = [FileMove(file_1, new_workspace_path_1), FileMove(file_2, new_workspace_path_2), FileMove(file_3, new_workspace_path_3), FileMove(file_4, new_workspace_path_4)] ScaleFile.objects.move_files(files) workspace_1.move_files.assert_called_once_with([FileMove(file_1, new_workspace_path_1), FileMove(file_2, new_workspace_path_2)]) workspace_2.move_files.assert_called_once_with([FileMove(file_3, new_workspace_path_3), FileMove(file_4, new_workspace_path_4)])
def test_successfully(self, mock_copy, mock_chmod, mock_exists, mock_makedirs): """Tests calling HostBroker.upload_files() successfully""" def new_exists(path): return False mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1) full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2) file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1) file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2) file_1_up = FileUpload(file_1, local_path_file_1) file_2_up = FileUpload(file_2, local_path_file_2) # Call method to test self.broker.upload_files(volume_path, [file_1_up, file_2_up]) # Check results two_calls = [call(os.path.dirname(full_workspace_path_file_1), mode=0755), call(os.path.dirname(full_workspace_path_file_2), mode=0755)] mock_makedirs.assert_has_calls(two_calls) two_calls = [call(local_path_file_1, full_workspace_path_file_1), call(local_path_file_2, full_workspace_path_file_2)] mock_copy.assert_has_calls(two_calls) two_calls = [call(full_workspace_path_file_1, 0644), call(full_workspace_path_file_2, 0644)] mock_chmod.assert_has_calls(two_calls)
def test_delete_file(self, mock_remove, mock_exists): """Tests removing a file""" def new_exists(path): return True mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file.json') full_path_file_1 = os.path.join(volume_path, file_path_1) full_path_file_2 = os.path.join(volume_path, file_path_2) file_1 = storage_test_utils.create_file(file_path=file_path_1) file_2 = storage_test_utils.create_file(file_path=file_path_2) # Call function test_1 = delete_files([file_1], volume_path, self.broker) self.assertEqual(test_1, None) test_2 = delete_files([file_2], volume_path, self.broker) self.assertEqual(test_2, None) # Check results two_calls = [call(full_path_file_1), call(full_path_file_2)] mock_remove.assert_has_calls(two_calls)
def test_successfully(self, mock_remove, mock_exists): """Tests calling HostBroker.delete_files() successfully""" def new_exists(path): return True mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file.json') full_path_file_1 = os.path.join(volume_path, file_path_1) full_path_file_2 = os.path.join(volume_path, file_path_2) file_1 = storage_test_utils.create_file(file_path=file_path_1) file_2 = storage_test_utils.create_file(file_path=file_path_2) # Call method to test self.broker.delete_files(volume_path, [file_1, file_2]) # Check results two_calls = [call(full_path_file_1), call(full_path_file_2)] mock_remove.assert_has_calls(two_calls) self.assertTrue(file_1.is_deleted) self.assertIsNotNone(file_1.deleted) self.assertTrue(file_2.is_deleted) self.assertIsNotNone(file_2.deleted)
def test_deleted_file(self): '''Tests calling ScaleFileManager.download_files() with a deleted file''' download_dir = os.path.join('download', 'dir') work_dir = os.path.join('work', 'dir') workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace_1) local_path_1 = u'my/local/path/file.txt' file_2 = storage_test_utils.create_file(workspace=workspace_1) local_path_2 = u'another/local/path/file.txt' file_2.is_deleted = True file_2.save() file_3 = storage_test_utils.create_file(workspace=workspace_1) local_path_3 = u'another/local/path/file.json' workspace_1.download_files = MagicMock() workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1) workspace_2 = storage_test_utils.create_workspace() file_4 = storage_test_utils.create_file(workspace=workspace_2) local_path_4 = u'my/local/path/4/file.txt' file_5 = storage_test_utils.create_file(workspace=workspace_2) local_path_5 = u'another/local/path/5/file.txt' workspace_2.download_files = MagicMock() workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2) files = [(file_1, local_path_1), (file_2, local_path_2), (file_3, local_path_3), (file_4, local_path_4), (file_5, local_path_5)] self.assertRaises(DeletedFile, ScaleFile.objects.download_files, download_dir, work_dir, files)
def test_inactive_workspace(self): """Tests calling ScaleFileManager.download_files() with an inactive workspace""" workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace_1) local_path_1 = '/my/local/path/file.txt' file_2 = storage_test_utils.create_file(workspace=workspace_1) local_path_2 = '/another/local/path/file.txt' file_3 = storage_test_utils.create_file(workspace=workspace_1) local_path_3 = '/another/local/path/file.json' workspace_1.download_files = MagicMock() workspace_2 = storage_test_utils.create_workspace() workspace_2.is_active = False workspace_2.save() file_4 = storage_test_utils.create_file(workspace=workspace_2) local_path_4 = '/my/local/path/4/file.txt' file_5 = storage_test_utils.create_file(workspace=workspace_2) local_path_5 = '/another/local/path/5/file.txt' files = [ FileDownload(file_1, local_path_1), FileDownload(file_2, local_path_2), FileDownload(file_3, local_path_3), FileDownload(file_4, local_path_4), FileDownload(file_5, local_path_5) ] self.assertRaises(ArchivedWorkspace, ScaleFile.objects.download_files, files)
def test_successfully(self, mock_execute, mock_exists): """Tests calling NfsBroker.download_files() successfully""" def new_exists(path): return False mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1) full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2) file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1) file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2) file_1_dl = FileDownload(file_1, local_path_file_1, False) file_2_dl = FileDownload(file_2, local_path_file_2, False) # Call method to test self.broker.download_files(volume_path, [file_1_dl, file_2_dl]) # Check results two_calls = [call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]), call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])] mock_execute.assert_has_calls(two_calls)
def test_create_message(self): """ Tests calling the create message function for DeleteFiles """ job = job_test_utils.create_job() job_exe = job_test_utils.create_job_exe(job=job) trigger = trigger_test_utils.create_trigger_event() file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file1.json') file_path_3 = os.path.join('my_dir', 'my_file2.json') file_path_4 = os.path.join('my_dir', 'my_file3.json') file_1 = storage_test_utils.create_file(file_path=file_path_1, job_exe=job_exe) file_2 = storage_test_utils.create_file(file_path=file_path_2, job_exe=job_exe) file_3 = storage_test_utils.create_file(file_path=file_path_3, job_exe=job_exe) file_4 = storage_test_utils.create_file(file_path=file_path_4, job_exe=job_exe) files = [file_1, file_2, file_3, file_4] # No exception is success create_delete_files_messages(files=files, job_id=job.id, trigger_id=trigger.id, source_file_id=self.source_file.id, purge=True)
def test_upload_files(self, mock_client_class): """Tests uploading files successfully""" s3_object_1 = MagicMock() s3_object_2 = MagicMock() mock_client = MagicMock(S3Client) mock_client.get_object.side_effect = [s3_object_1, s3_object_2] mock_client_class.return_value.__enter__ = Mock(return_value=mock_client) file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1, media_type='text/plain') file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2, media_type='application/json') file_1_up = FileUpload(file_1, local_path_file_1) file_2_up = FileUpload(file_2, local_path_file_2) # Call method to test mo = mock_open() with patch('__builtin__.open', mo, create=True): self.broker.upload_files(None, [file_1_up, file_2_up]) # Check results self.assertTrue(s3_object_1.upload_file.called) self.assertTrue(s3_object_2.upload_file.called) self.assertEqual(s3_object_1.upload_file.call_args[0][1]['ContentType'], 'text/plain') self.assertEqual(s3_object_2.upload_file.call_args[0][1]['ContentType'], 'application/json')
def test_no_files(self): '''Tests calling ScaleFileManager.get_total_file_size() where no files match the file IDs''' storage_test_utils.create_file(file_size=100) file_size = ScaleFile.objects.get_total_file_size([4444444444, 555555555555, 666666666666]) self.assertEqual(file_size, 0)
def test_delete_files(self, mock_client_class): """Tests deleting files successfully""" s3_object_1 = MagicMock() s3_object_2 = MagicMock() mock_client = MagicMock(S3Client) mock_client.get_object.side_effect = [s3_object_1, s3_object_2] mock_client_class.return_value.__enter__ = Mock(return_value=mock_client) file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file.json') file_1 = storage_test_utils.create_file(file_path=file_path_1) file_2 = storage_test_utils.create_file(file_path=file_path_2) # Call method to test self.broker.delete_files(None, [file_1, file_2]) # Check results self.assertTrue(s3_object_1.delete.called) self.assertTrue(s3_object_2.delete.called) self.assertTrue(file_1.is_deleted) self.assertIsNotNone(file_1.deleted) self.assertTrue(file_2.is_deleted) self.assertIsNotNone(file_2.deleted)
def test_deleted_file(self): '''Tests calling ScaleFileManager.download_files() with a deleted file''' download_dir = os.path.join('download', 'dir') work_dir = os.path.join('work', 'dir') workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace_1) local_path_1 = u'my/local/path/file.txt' file_2 = storage_test_utils.create_file(workspace=workspace_1) local_path_2 = u'another/local/path/file.txt' file_2.is_deleted = True file_2.save() file_3 = storage_test_utils.create_file(workspace=workspace_1) local_path_3 = u'another/local/path/file.json' workspace_1.download_files = MagicMock() workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir( work_dir, workspace_1) workspace_2 = storage_test_utils.create_workspace() file_4 = storage_test_utils.create_file(workspace=workspace_2) local_path_4 = u'my/local/path/4/file.txt' file_5 = storage_test_utils.create_file(workspace=workspace_2) local_path_5 = u'another/local/path/5/file.txt' workspace_2.download_files = MagicMock() workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir( work_dir, workspace_2) files = [(file_1, local_path_1), (file_2, local_path_2), (file_3, local_path_3), (file_4, local_path_4), (file_5, local_path_5)] self.assertRaises(DeletedFile, ScaleFile.objects.download_files, download_dir, work_dir, files)
def test_download_files(self, mock_client_class): """Tests downloading files successfully""" s3_object_1 = MagicMock() s3_object_2 = MagicMock() mock_client = MagicMock(S3Client) mock_client.get_object.side_effect = [s3_object_1, s3_object_2] mock_client_class.return_value.__enter__ = Mock(return_value=mock_client) file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1) file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2) file_1_dl = FileDownload(file_1, local_path_file_1) file_2_dl = FileDownload(file_2, local_path_file_2) # Call method to test mo = mock_open() with patch('__builtin__.open', mo, create=True): self.broker.download_files(None, [file_1_dl, file_2_dl]) # Check results self.assertTrue(s3_object_1.download_file.called) self.assertTrue(s3_object_2.download_file.called)
def test_successfully(self, mock_remove, mock_exists): """Tests calling NfsBroker.delete_files() successfully""" def new_exists(path): return True mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file.json') full_path_file_1 = os.path.join(volume_path, file_path_1) full_path_file_2 = os.path.join(volume_path, file_path_2) file_1 = storage_test_utils.create_file(file_path=file_path_1) file_2 = storage_test_utils.create_file(file_path=file_path_2) # Call method to test self.broker.delete_files(volume_path, [file_1, file_2]) # Check results two_calls = [call(full_path_file_1), call(full_path_file_2)] mock_remove.assert_has_calls(two_calls) self.assertTrue(file_1.is_deleted) self.assertIsNotNone(file_1.deleted) self.assertTrue(file_2.is_deleted) self.assertIsNotNone(file_2.deleted)
def test_move_files(self, mock_client_class): """Tests moving files successfully""" s3_object_1a = MagicMock() s3_object_1b = MagicMock() s3_object_2a = MagicMock() s3_object_2b = MagicMock() mock_client = MagicMock(S3Client) mock_client.get_object.side_effect = [s3_object_1a, s3_object_1b, s3_object_2a, s3_object_2b] mock_client_class.return_value.__enter__ = Mock(return_value=mock_client) file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' old_workspace_path_1 = os.path.join('my_dir_1', file_name_1) old_workspace_path_2 = os.path.join('my_dir_2', file_name_2) new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1) new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2) file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1) file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2) file_1_mv = FileMove(file_1, new_workspace_path_1) file_2_mv = FileMove(file_2, new_workspace_path_2) # Call method to test self.broker.move_files(None, [file_1_mv, file_2_mv]) # Check results self.assertTrue(s3_object_1b.copy_from.called) self.assertTrue(s3_object_1a.delete.called) self.assertTrue(s3_object_2b.copy_from.called) self.assertTrue(s3_object_2a.delete.called) self.assertEqual(file_1.file_path, new_workspace_path_1) self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_host_link_files(self, mock_execute, mock_client_class): """Tests sym-linking files successfully""" volume_path = os.path.join('the', 'volume', 'path') file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) full_workspace_path_file_1 = os.path.join(volume_path, workspace_path_file_1) full_workspace_path_file_2 = os.path.join(volume_path, workspace_path_file_2) file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1) file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2) file_1_dl = FileDownload(file_1, local_path_file_1, True) file_2_dl = FileDownload(file_2, local_path_file_2, True) # Call method to test self.broker.download_files(volume_path, [file_1_dl, file_2_dl]) # Check results two_calls = [call(['ln', '-s', full_workspace_path_file_1, local_path_file_1]), call(['ln', '-s', full_workspace_path_file_2, local_path_file_2])] mock_execute.assert_has_calls(two_calls)
def test_success(self): """Tests calling ScaleFileManager.download_files() successfully""" workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace_1) local_path_1 = '/my/local/path/file.txt' file_2 = storage_test_utils.create_file(workspace=workspace_1) local_path_2 = '/another/local/path/file.txt' file_3 = storage_test_utils.create_file(workspace=workspace_1) local_path_3 = '/another/local/path/file.json' workspace_1.setup_download_dir = MagicMock() workspace_1.download_files = MagicMock() workspace_2 = storage_test_utils.create_workspace() file_4 = storage_test_utils.create_file(workspace=workspace_2) local_path_4 = '/my/local/path/4/file.txt' file_5 = storage_test_utils.create_file(workspace=workspace_2) local_path_5 = '/another/local/path/5/file.txt' workspace_2.setup_download_dir = MagicMock() workspace_2.download_files = MagicMock() files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False), FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False), FileDownload(file_5, local_path_5, False)] ScaleFile.objects.download_files(files) workspace_1.download_files.assert_called_once_with([FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False), FileDownload(file_3, local_path_3, False)]) workspace_2.download_files.assert_called_once_with([FileDownload(file_4, local_path_4, False), FileDownload(file_5, local_path_5, False)])
def test_deleted_file(self): '''Tests calling ScaleFileManager.move_files() with a deleted file''' work_dir = os.path.join('work', 'dir') workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1) new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path)) file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1) file_2.is_deleted = True file_2.save() new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path)) workspace_1.move_files = MagicMock() workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1) workspace_2 = storage_test_utils.create_workspace() workspace_2.is_active = False workspace_2.save() file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2) new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path)) file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2) new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path)) workspace_2.move_files = MagicMock() workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2) files = [(file_1, new_workspace_path_1), (file_2, new_workspace_path_2), (file_3, new_workspace_path_3), (file_4, new_workspace_path_4)] self.assertRaises(DeletedFile, ScaleFile.objects.move_files, work_dir, files)
def test_deleted_file(self): """Tests calling ScaleFileManager.download_files() with a deleted file""" workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace_1) local_path_1 = '/my/local/path/file.txt' file_2 = storage_test_utils.create_file(workspace=workspace_1) local_path_2 = '/another/local/path/file.txt' file_2.is_deleted = True file_2.save() file_3 = storage_test_utils.create_file(workspace=workspace_1) local_path_3 = '/another/local/path/file.json' workspace_1.download_files = MagicMock() workspace_2 = storage_test_utils.create_workspace() file_4 = storage_test_utils.create_file(workspace=workspace_2) local_path_4 = '/my/local/path/4/file.txt' file_5 = storage_test_utils.create_file(workspace=workspace_2) local_path_5 = '/another/local/path/5/file.txt' workspace_2.download_files = MagicMock() files = [FileDownload(file_1, local_path_1, False), FileDownload(file_2, local_path_2, False), FileDownload(file_3, local_path_3, False), FileDownload(file_4, local_path_4, False), FileDownload(file_5, local_path_5, False)] self.assertRaises(DeletedFile, ScaleFile.objects.download_files, files)
def test_inactive_workspace(self): """Tests calling ScaleFileManager.move_files() with an inactive workspace""" workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1) new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path)) file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1) new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path)) workspace_1.move_files = MagicMock() workspace_2 = storage_test_utils.create_workspace() workspace_2.is_active = False workspace_2.save() file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2) new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path)) file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2) new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path)) workspace_2.move_files = MagicMock() files = [ FileMove(file_1, new_workspace_path_1), FileMove(file_2, new_workspace_path_2), FileMove(file_3, new_workspace_path_3), FileMove(file_4, new_workspace_path_4) ] self.assertRaises(ArchivedWorkspace, ScaleFile.objects.move_files, files)
def test_download_files(self, mock_client_class): """Tests downloading files successfully""" s3_object_1 = MagicMock() s3_object_2 = MagicMock() mock_client = MagicMock(S3Client) mock_client.get_object.side_effect = [s3_object_1, s3_object_2] mock_client_class.return_value.__enter__ = Mock(return_value=mock_client) file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' local_path_file_1 = os.path.join('my_dir_1', file_name_1) local_path_file_2 = os.path.join('my_dir_2', file_name_2) workspace_path_file_1 = os.path.join('my_wrk_dir_1', file_name_1) workspace_path_file_2 = os.path.join('my_wrk_dir_2', file_name_2) file_1 = storage_test_utils.create_file(file_path=workspace_path_file_1) file_2 = storage_test_utils.create_file(file_path=workspace_path_file_2) file_1_dl = FileDownload(file_1, local_path_file_1, False) file_2_dl = FileDownload(file_2, local_path_file_2, False) # Call method to test mo = mock_open() with patch('__builtin__.open', mo, create=True): self.broker.download_files(None, [file_1_dl, file_2_dl]) # Check results self.assertTrue(s3_object_1.download_file.called) self.assertTrue(s3_object_2.download_file.called)
def setUp(self): django.setup() self.file_1 = storage_utils.create_file(file_name='my_json_file.json', media_type='application/json') self.file_2 = storage_utils.create_file(file_name='my_text_file_1.txt', media_type='text/plain') self.file_3 = storage_utils.create_file(file_name='my_text_file_2.txt', media_type='text/plain')
def test_no_files(self): '''Tests calling ScaleFileManager.get_total_file_size() where no files match the file IDs''' storage_test_utils.create_file(file_size=100) file_size = ScaleFile.objects.get_total_file_size( [4444444444, 555555555555, 666666666666]) self.assertEqual(file_size, 0)
def test_success(self): '''Tests calling ScaleFileManager.get_total_file_size() successfully''' file_1 = storage_test_utils.create_file(file_size=100) file_2 = storage_test_utils.create_file(file_size=300) storage_test_utils.create_file(file_size=700) file_size = ScaleFile.objects.get_total_file_size([file_1.id, file_2.id]) self.assertEqual(file_size, 400)
def setUp(self): django.setup() self.job_1 = job_test_utils.create_job() self.trigger_1 = trigger_test_utils.create_trigger_event() self.job_exe_1 = job_test_utils.create_job_exe(job=self.job_1) self.file_1 = storage_test_utils.create_file(job_exe=self.job_exe_1) self.source_file = storage_test_utils.create_file(file_type='SOURCE') self.workspace = storage_test_utils.create_workspace()
def test_success(self, mock_makedirs): '''Tests calling ScaleFileManager.move_files() successfully''' work_dir = os.path.join('work', 'dir') workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1) old_path_1 = file_1.file_path new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path)) file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1) old_path_2 = file_2.file_path new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path)) workspace_1.move_files = MagicMock() workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir( work_dir, workspace_1) workspace_2 = storage_test_utils.create_workspace() file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2) old_path_3 = file_3.file_path new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path)) file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2) old_path_4 = file_4.file_path new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path)) workspace_2.move_files = MagicMock() workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir( work_dir, workspace_2) files = [ (file_1, new_workspace_path_1), (file_2, new_workspace_path_2), (file_3, new_workspace_path_3), (file_4, new_workspace_path_4) ] ScaleFile.objects.move_files(work_dir, files) workspace_1.move_files.assert_called_once_with( workspace_1_work_dir, [(old_path_1, new_workspace_path_1), (old_path_2, new_workspace_path_2)]) workspace_2.move_files.assert_called_once_with( workspace_2_work_dir, [(old_path_3, new_workspace_path_3), (old_path_4, new_workspace_path_4)]) # Check models for new workspace paths new_file_1 = ScaleFile.objects.get(id=file_1.id) self.assertEqual(new_file_1.file_path, new_workspace_path_1) new_file_2 = ScaleFile.objects.get(id=file_2.id) self.assertEqual(new_file_2.file_path, new_workspace_path_2) new_file_3 = ScaleFile.objects.get(id=file_3.id) self.assertEqual(new_file_3.file_path, new_workspace_path_3) new_file_4 = ScaleFile.objects.get(id=file_4.id) self.assertEqual(new_file_4.file_path, new_workspace_path_4)
def test_success(self): '''Tests calling ScaleFileManager.get_total_file_size() successfully''' file_1 = storage_test_utils.create_file(file_size=100) file_2 = storage_test_utils.create_file(file_size=300) storage_test_utils.create_file(file_size=700) file_size = ScaleFile.objects.get_total_file_size( [file_1.id, file_2.id]) self.assertEqual(file_size, 400)
def test_execute_force_stop(self): """Tests calling DeleteFile.execute() successfully""" # Create PurgeResults entry source_file = storage_test_utils.create_file() trigger = trigger_test_utils.create_trigger_event() PurgeResults.objects.create(source_file_id=source_file.id, trigger_event=trigger, force_stop_purge=True) self.assertEqual( PurgeResults.objects.values_list( 'num_products_deleted', flat=True).get(source_file_id=source_file.id), 0) job = job_test_utils.create_job() job_exe = job_test_utils.create_job_exe(job=job) file_path_1 = os.path.join('my_dir', 'my_file.txt') file_path_2 = os.path.join('my_dir', 'my_file1.json') file_path_3 = os.path.join('my_dir', 'my_file2.json') file_path_4 = os.path.join('my_dir', 'my_file3.json') file_1 = storage_test_utils.create_file(file_path=file_path_1, job_exe=job_exe) file_2 = storage_test_utils.create_file(file_path=file_path_2, job_exe=job_exe) file_3 = storage_test_utils.create_file(file_path=file_path_3, job_exe=job_exe) file_4 = storage_test_utils.create_file(file_path=file_path_4, job_exe=job_exe) # Add files to message message = DeleteFiles() message.purge = True message.job_id = job.id message.source_file_id = source_file.id message.trigger_id = trigger.id if message.can_fit_more(): message.add_file(file_1.id) if message.can_fit_more(): message.add_file(file_2.id) if message.can_fit_more(): message.add_file(file_3.id) if message.can_fit_more(): message.add_file(file_4.id) # Execute message result = message.execute() self.assertTrue(result) # Check results are accurate self.assertEqual( PurgeResults.objects.values_list( 'num_products_deleted', flat=True).get(source_file_id=source_file.id), 0)
def test_schedule_date_range_data_ended(self): """Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction""" file1 = storage_test_utils.create_file() file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc) file1.data_ended = datetime.datetime(2016, 1, 10, tzinfo=utc) file1.save() data1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file1.id, }], 'workspace_id': self.workspace.id, } recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data1), event=self.event).recipe file2 = storage_test_utils.create_file() file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc) file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc) file2.save() data2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file2.id, }], 'workspace_id': self.workspace.id, } Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data2), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'ended': '2016-01-15T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def test_inactive_workspace(self): """Tests calling deleting files from an inactive workspace""" workspace_1 = storage_test_utils.create_workspace() workspace_1.download_files = MagicMock() file_1 = storage_test_utils.create_file(workspace=workspace_1) workspace_2 = storage_test_utils.create_workspace(is_active=False) file_2 = storage_test_utils.create_file(workspace=workspace_2) files = [file_1, file_2] self.assertRaises(ArchivedWorkspace, ScaleFile.objects.delete_files, files)
def test_successfully(self, mock_move, mock_chmod, mock_exists, mock_makedirs): """Tests calling HostBroker.move_files() successfully""" def new_exists(path): return path.count('new') == 0 mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' old_workspace_path_1 = os.path.join('my_dir_1', file_name_1) old_workspace_path_2 = os.path.join('my_dir_2', file_name_2) new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1) new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2) full_old_workspace_path_1 = os.path.join(volume_path, old_workspace_path_1) full_old_workspace_path_2 = os.path.join(volume_path, old_workspace_path_2) full_new_workspace_path_1 = os.path.join(volume_path, new_workspace_path_1) full_new_workspace_path_2 = os.path.join(volume_path, new_workspace_path_2) file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1) file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2) file_1_mv = FileMove(file_1, new_workspace_path_1) file_2_mv = FileMove(file_2, new_workspace_path_2) # Call method to test self.broker.move_files(volume_path, [file_1_mv, file_2_mv]) # Check results two_calls = [ call(os.path.dirname(full_new_workspace_path_1), mode=0755), call(os.path.dirname(full_new_workspace_path_2), mode=0755) ] mock_makedirs.assert_has_calls(two_calls) two_calls = [ call(full_old_workspace_path_1, full_new_workspace_path_1), call(full_old_workspace_path_2, full_new_workspace_path_2) ] mock_move.assert_has_calls(two_calls) two_calls = [ call(full_new_workspace_path_1, 0644), call(full_new_workspace_path_2, 0644) ] mock_chmod.assert_has_calls(two_calls) self.assertEqual(file_1.file_path, new_workspace_path_1) self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_schedule_date_range_data_ended(self): """Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction""" file1 = storage_test_utils.create_file() file1.data_started = datetime.datetime(2016, 1, 1) file1.data_ended = datetime.datetime(2016, 1, 10) file1.save() data1 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file1.id, }], 'workspace_id': self.workspace.id, } recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data1), event=self.event).recipe file2 = storage_test_utils.create_file() file2.data_started = datetime.datetime(2016, 2, 1) file2.data_ended = datetime.datetime(2016, 2, 10) file2.save() data2 = { 'version': '1.0', 'input_data': [{ 'name': 'Recipe Input', 'file_id': file2.id, }], 'workspace_id': self.workspace.id, } Recipe.objects.create_recipe(recipe_type=self.recipe_type, data=RecipeData(data2), event=self.event) recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2) definition = { 'date_range': { 'type': 'data', 'ended': '2016-01-15T00:00:00.000Z', }, } batch = batch_test_utils.create_batch(recipe_type=self.recipe_type, definition=definition) Batch.objects.schedule_recipes(batch.id) batch = Batch.objects.get(pk=batch.id) self.assertEqual(batch.status, 'CREATED') self.assertEqual(batch.created_count, 1) self.assertEqual(batch.total_count, 1) batch_recipes = BatchRecipe.objects.all() self.assertEqual(len(batch_recipes), 1) self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def setUp(self): django.setup() self.workspace = storage_utils.create_workspace() self.file_name_1 = 'my_file.txt' self.media_type_1 = 'text/plain' self.product_file_1 = storage_utils.create_file(file_name=self.file_name_1, media_type=self.media_type_1, workspace=self.workspace) self.file_name_2 = 'my_file.txt' self.media_type_2 = 'text/plain' self.product_file_2 = storage_utils.create_file(file_name=self.file_name_2, media_type=self.media_type_2, workspace=self.workspace) self.invalid_product_file_id = long(999)
def setUp(self): django.setup() self.count = 1 self.job_type = job_test_utils.create_seed_job_type() self.job = job_test_utils.create_job(job_type=self.job_type) self.job_exe = job_test_utils.create_job_exe(status='COMPLETED', job=self.job) self.wp1 = storage_test_utils.create_workspace() self.wp2 = storage_test_utils.create_workspace() self.prod1 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp1, job_exe=self.job_exe) self.prod2 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp1, job_exe=self.job_exe) self.prod3 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp2, job_exe=self.job_exe) self.file_1 = storage_test_utils.create_file(file_type='SOURCE') self.event = trigger_test_utils.create_trigger_event() PurgeResults.objects.create(source_file_id=self.file_1.id, trigger_event=self.event)
def test_success(self): """Tests deleting files successfully""" workspace_1 = storage_test_utils.create_workspace() workspace_1.delete_files = MagicMock() file_1 = storage_test_utils.create_file(workspace=workspace_1) workspace_2 = storage_test_utils.create_workspace() workspace_2.delete_files = MagicMock() file_2 = storage_test_utils.create_file(workspace=workspace_2) files = [file_1, file_2] ScaleFile.objects.delete_files(files) workspace_1.delete_files.assert_called_once_with([file_1]) workspace_2.delete_files.assert_called_once_with([file_2])
def test_url_file_slash(self): '''Tests building a URL for a file where the file path URL has a leading slash.''' ws = storage_test_utils.create_workspace(name='test', base_url='http://localhost') file = storage_test_utils.create_file(file_name='test.txt', file_path='/file/path/test.txt', workspace=ws) self.assertEqual(file.url, 'http://localhost/file/path/test.txt')
def test_inputs_and_products(self): """Tests creating links for inputs and then later replacing with generated products.""" file_8 = storage_test_utils.create_file() parent_ids = [self.file_4.id, self.file_6.id, self.file_7.id] child_ids = [file_8.id] job_exe = job_test_utils.create_job_exe() recipe_test_utils.create_recipe_job(job=job_exe.job) # First create only the input files FileAncestryLink.objects.create_file_ancestry_links(parent_ids, None, job_exe) # Replace the inputs with the new links for both inputs and products FileAncestryLink.objects.create_file_ancestry_links(parent_ids, child_ids, job_exe) # Make sure the old entries were deleted old_direct_qry = FileAncestryLink.objects.filter(descendant__isnull=True, job_exe=job_exe, ancestor_job__isnull=True) self.assertEqual(len(old_direct_qry), 0) old_indirect_qry = FileAncestryLink.objects.filter(descendant__isnull=True, job_exe=job_exe, ancestor_job__isnull=False) self.assertEqual(len(old_indirect_qry), 0) direct_qry = FileAncestryLink.objects.filter(descendant=file_8, job_exe=job_exe, ancestor_job__isnull=True) self.assertEqual(direct_qry.count(), 3) file_8_parent_ids = {link.ancestor_id for link in direct_qry} self.assertSetEqual(file_8_parent_ids, {self.file_4.id, self.file_6.id, self.file_7.id}) indirect_qry = FileAncestryLink.objects.filter(descendant=file_8, job_exe=job_exe, ancestor_job__isnull=False) self.assertEqual(indirect_qry.count(), 3) file_8_ancestor_ids = {link.ancestor_id for link in indirect_qry} self.assertSetEqual(file_8_ancestor_ids, {self.file_1.id, self.file_2.id, self.file_3.id})
def test_deleted_file(self): """Tests attempting to delete a file that is already deleted""" workspace_1 = storage_test_utils.create_workspace() workspace_1.delete_files = MagicMock() file_1a = storage_test_utils.create_file(workspace=workspace_1) file_1b = storage_test_utils.create_file(workspace=workspace_1, is_deleted=True) workspace_2 = storage_test_utils.create_workspace() workspace_2.delete_files = MagicMock() file_2 = storage_test_utils.create_file(workspace=workspace_2) files = [file_1a, file_1b, file_2] ScaleFile.objects.delete_files(files) workspace_1.delete_files.assert_called_once_with([file_1a, file_1b]) workspace_2.delete_files.assert_called_once_with([file_2])
def setUp(self): django.setup() # Generation 1 self.file_1 = storage_test_utils.create_file() self.file_2 = storage_test_utils.create_file() # Generation 2 job_exe_1 = job_test_utils.create_job_exe() recipe_job_1 = recipe_test_utils.create_recipe_job(job=job_exe_1.job) self.file_3 = prod_test_utils.create_product(job_exe=job_exe_1) self.file_4 = prod_test_utils.create_product(job_exe=job_exe_1) self.file_5 = prod_test_utils.create_product(job_exe=job_exe_1) # Generation 3 job_exe_2 = job_test_utils.create_job_exe() recipe_job_2 = recipe_test_utils.create_recipe_job(job=job_exe_2.job) self.file_6 = prod_test_utils.create_product(job_exe=job_exe_2) # Stand alone file self.file_7 = prod_test_utils.create_product() # First job links generation 1 to 2 FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_3, job_exe=job_exe_1, job=job_exe_1.job, recipe=recipe_job_1.recipe) FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_4, job_exe=job_exe_1, job=job_exe_1.job, recipe=recipe_job_1.recipe) FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_5, job_exe=job_exe_1, job=job_exe_1.job, recipe=recipe_job_1.recipe) FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_3, job_exe=job_exe_1, job=job_exe_1.job, recipe=recipe_job_1.recipe) FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_4, job_exe=job_exe_1, job=job_exe_1.job, recipe=recipe_job_1.recipe) FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_5, job_exe=job_exe_1, job=job_exe_1.job, recipe=recipe_job_1.recipe) # Second job links generation 2 to 3 FileAncestryLink.objects.create(ancestor=self.file_3, descendant=self.file_6, job_exe=job_exe_2, job=job_exe_2.job, recipe=recipe_job_2.recipe) FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_6, job_exe=job_exe_2, job=job_exe_2.job, recipe=recipe_job_2.recipe, ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job) FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_6, job_exe=job_exe_2, job=job_exe_2.job, recipe=recipe_job_2.recipe, ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
def test_set_deleted(self): """Tests marking a file as deleted.""" scale_file = storage_test_utils.create_file() scale_file.set_deleted() self.assertTrue(scale_file.is_deleted) self.assertIsNotNone(scale_file.deleted)
def test_success(self): '''Tests calling ScaleFileManager.move_files() successfully''' work_dir = os.path.join('work', 'dir') workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1) old_path_1 = file_1.file_path new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path)) file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1) old_path_2 = file_2.file_path new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path)) workspace_1.move_files = MagicMock() workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1) workspace_2 = storage_test_utils.create_workspace() file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2) old_path_3 = file_3.file_path new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path)) file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2) old_path_4 = file_4.file_path new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path)) workspace_2.move_files = MagicMock() workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2) files = [(file_1, new_workspace_path_1), (file_2, new_workspace_path_2), (file_3, new_workspace_path_3), (file_4, new_workspace_path_4)] ScaleFile.objects.move_files(work_dir, files) workspace_1.move_files.assert_called_once_with(workspace_1_work_dir, [(old_path_1, new_workspace_path_1), (old_path_2, new_workspace_path_2)]) workspace_2.move_files.assert_called_once_with(workspace_2_work_dir, [(old_path_3, new_workspace_path_3), (old_path_4, new_workspace_path_4)]) # Check models for new workspace paths new_file_1 = ScaleFile.objects.get(id=file_1.id) self.assertEqual(new_file_1.file_path, new_workspace_path_1) new_file_2 = ScaleFile.objects.get(id=file_2.id) self.assertEqual(new_file_2.file_path, new_workspace_path_2) new_file_3 = ScaleFile.objects.get(id=file_3.id) self.assertEqual(new_file_3.file_path, new_workspace_path_3) new_file_4 = ScaleFile.objects.get(id=file_4.id) self.assertEqual(new_file_4.file_path, new_workspace_path_4)
def setUp(self): django.setup() self.input_name_1 = 'Test Input 1' self.output_name_1 = 'Test Output 1' interface_1 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name_1, 'type': 'file', 'media_types': ['text/plain'], }], 'output_data': [{ 'name': self.output_name_1, 'type': 'files', 'media_type': 'image/png', }], } self.job_type_1 = job_test_utils.create_job_type(interface=interface_1) self.input_name_2 = 'Test Input 2' self.output_name_2 = 'Test Output 2' interface_2 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name_2, 'type': 'files', 'media_types': ['image/png', 'image/tiff'], }], 'output_data': [{ 'name': self.output_name_2, 'type': 'file', }], } self.job_type_2 = job_test_utils.create_job_type(interface=interface_2) self.input_name_3 = 'Test Input 3' self.output_name_3 = 'Test Output 3' interface_3 = { 'version': '1.0', 'command': 'my_cmd', 'command_arguments': 'args', 'input_data': [{ 'name': self.input_name_3, 'type': 'file', 'media_types': ['text/plain'], }], } self.job_type_3 = job_test_utils.create_job_type(interface=interface_3) self.file_1 = storage_test_utils.create_file(media_type='text/plain')
def test_successfully(self, mock_move, mock_chmod, mock_exists, mock_makedirs): """Tests calling NfsBroker.move_files() successfully""" def new_exists(path): return False mock_exists.side_effect = new_exists volume_path = os.path.join('the', 'volume', 'path') file_name_1 = 'my_file.txt' file_name_2 = 'my_file.json' old_workspace_path_1 = os.path.join('my_dir_1', file_name_1) old_workspace_path_2 = os.path.join('my_dir_2', file_name_2) new_workspace_path_1 = os.path.join('my_new_dir_1', file_name_1) new_workspace_path_2 = os.path.join('my_new_dir_2', file_name_2) full_old_workspace_path_1 = os.path.join(volume_path, old_workspace_path_1) full_old_workspace_path_2 = os.path.join(volume_path, old_workspace_path_2) full_new_workspace_path_1 = os.path.join(volume_path, new_workspace_path_1) full_new_workspace_path_2 = os.path.join(volume_path, new_workspace_path_2) file_1 = storage_test_utils.create_file(file_path=old_workspace_path_1) file_2 = storage_test_utils.create_file(file_path=old_workspace_path_2) file_1_mv = FileMove(file_1, new_workspace_path_1) file_2_mv = FileMove(file_2, new_workspace_path_2) # Call method to test self.broker.move_files(volume_path, [file_1_mv, file_2_mv]) # Check results two_calls = [call(os.path.dirname(full_new_workspace_path_1), mode=0755), call(os.path.dirname(full_new_workspace_path_2), mode=0755)] mock_makedirs.assert_has_calls(two_calls) two_calls = [call(full_old_workspace_path_1, full_new_workspace_path_1), call(full_old_workspace_path_2, full_new_workspace_path_2)] mock_move.assert_has_calls(two_calls) two_calls = [call(full_new_workspace_path_1, 0644), call(full_new_workspace_path_2, 0644)] mock_chmod.assert_has_calls(two_calls) self.assertEqual(file_1.file_path, new_workspace_path_1) self.assertEqual(file_2.file_path, new_workspace_path_2)
def test_inactive_workspace(self): """Tests calling ScaleFileManager.move_files() with an inactive workspace""" workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(file_name='my_file_1.txt', workspace=workspace_1) new_workspace_path_1 = os.path.join('my', 'new', 'path', '1', os.path.basename(file_1.file_path)) file_2 = storage_test_utils.create_file(file_name='my_file_2.txt', workspace=workspace_1) new_workspace_path_2 = os.path.join('my', 'new', 'path', '2', os.path.basename(file_2.file_path)) workspace_1.move_files = MagicMock() workspace_2 = storage_test_utils.create_workspace() workspace_2.is_active = False workspace_2.save() file_3 = storage_test_utils.create_file(file_name='my_file_3.txt', workspace=workspace_2) new_workspace_path_3 = os.path.join('my', 'new', 'path', '3', os.path.basename(file_3.file_path)) file_4 = storage_test_utils.create_file(file_name='my_file_4.txt', workspace=workspace_2) new_workspace_path_4 = os.path.join('my', 'new', 'path', '4', os.path.basename(file_4.file_path)) workspace_2.move_files = MagicMock() files = [FileMove(file_1, new_workspace_path_1), FileMove(file_2, new_workspace_path_2), FileMove(file_3, new_workspace_path_3), FileMove(file_4, new_workspace_path_4)] self.assertRaises(ArchivedWorkspace, ScaleFile.objects.move_files, files)
def test_success(self): '''Tests calling ScaleFileManager.download_files() successfully''' download_dir = os.path.join('download', 'dir') work_dir = os.path.join('work', 'dir') workspace_1 = storage_test_utils.create_workspace() file_1 = storage_test_utils.create_file(workspace=workspace_1) local_path_1 = u'my/local/path/file.txt' file_2 = storage_test_utils.create_file(workspace=workspace_1) local_path_2 = u'another/local/path/file.txt' file_3 = storage_test_utils.create_file(workspace=workspace_1) local_path_3 = u'another/local/path/file.json' workspace_1.setup_download_dir = MagicMock() workspace_1.download_files = MagicMock() workspace_1_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_1) workspace_2 = storage_test_utils.create_workspace() file_4 = storage_test_utils.create_file(workspace=workspace_2) local_path_4 = u'my/local/path/4/file.txt' file_5 = storage_test_utils.create_file(workspace=workspace_2) local_path_5 = u'another/local/path/5/file.txt' workspace_2.setup_download_dir = MagicMock() workspace_2.download_files = MagicMock() workspace_2_work_dir = ScaleFile.objects._get_workspace_work_dir(work_dir, workspace_2) files = [(file_1, local_path_1), (file_2, local_path_2), (file_3, local_path_3), (file_4, local_path_4), (file_5, local_path_5)] ScaleFile.objects.download_files(download_dir, work_dir, files) workspace_1.setup_download_dir.assert_called_once_with(download_dir, workspace_1_work_dir) workspace_1.download_files.assert_called_once_with(download_dir, workspace_1_work_dir, [(file_1.file_path, local_path_1), (file_2.file_path, local_path_2), (file_3.file_path, local_path_3)]) workspace_2.setup_download_dir.assert_called_once_with(download_dir, workspace_2_work_dir) workspace_2.download_files.assert_called_once_with(download_dir, workspace_2_work_dir, [(file_4.file_path, local_path_4), (file_5.file_path, local_path_5)])