def test_copy_s3_to_local(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.s3_gateway.download_file = MagicMock(return_value=None) service.copy(self.S3_FILE, self.LOCAL_FILE) service.s3_gateway.download_file.assert_called_with( "bucket", "test_file", str(self.LOCAL_FILE))
def test_copy_s3_dir_to_s3_ok(self, MockS3Gateway, os_makedirs): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.s3_gateway.object_exists = MagicMock(return_value=True) service.s3_gateway.list_object2 = MagicMock(return_value=self.S3_DIR) service.s3_gateway.put_object = MagicMock(return_value=None) service.s3_gateway.copy = MagicMock(return_value=None) service.copy(self.S3_FOLDER, self.S3_FOLDER_COPY, True) service.s3_gateway.put_object.assert_has_calls( [ call("bucket", "test_folder_copy/bar/", ""), call("bucket", "test_folder_copy/baz/", ""), ], any_order=True, ) service.s3_gateway.copy.assert_has_calls( [ call("bucket", "test_folder/baz/a", "bucket", "test_folder_copy/baz/a"), call("bucket", "test_folder/baz/b", "bucket", "test_folder_copy/baz/b"), ], any_order=True, )
def test_file_exists(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.file_exists(self.S3_FILE) service.s3_gateway.object_exists.assert_called_with( "bucket", "test_file")
def _download_executables( repository_path: str, package_name: str, ) -> None: s3_region = S3Path(repository_path).region team, exe_name = _parse_package_name(package_name) exe_local_path = DEFAULT_EXE_FOLDER + exe_name exe_s3_path = repository_path + package_name storage_svc = S3StorageService(s3_region) storage_svc.copy(exe_s3_path, exe_local_path)
def setUp(self): storage_svc = S3StorageService("us-west-1") self.s3_storage_repo = S3MPCInstanceRepository(storage_svc, TEST_BASE_DIR) self.mpc_instance = MPCInstance( instance_id=TEST_INSTANCE_ID, game_name=TEST_GAME_NAME, mpc_role=TEST_MPC_ROLE, num_workers=TEST_NUM_WORKERS, server_ips=TEST_SERVER_IPS, status=MPCInstanceStatus.CREATED, input_args=TEST_INPUT_ARGS, output_args=TEST_OUTPUT_ARGS, concurrency_args=TEST_CONCURRENCY_ARGS, input_directory=TEST_INPUT_DIRECTORY, output_directory=TEST_OUTPUT_DIRECTORY, )
def test_copy_local_dir_to_s3_recursive_true(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.s3_gateway.put_object = MagicMock(return_value=None) service.s3_gateway.upload_file = MagicMock(return_value=None) with patch("os.path.isdir", return_value=True): with patch("os.walk", return_value=self.LOCAL_DIR): service.copy(self.LOCAL_FOLDER, self.S3_FOLDER, True) service.s3_gateway.put_object.assert_called_with( "bucket", "test_folder/bar/", "") service.s3_gateway.upload_file.assert_has_calls( [ call("/foo/baz/a", "bucket", "test_folder/baz/a"), call("/foo/baz/b", "bucket", "test_folder/baz/b"), ], any_order=True, )
def test_copy_s3_dir_to_local_ok(self, MockS3Gateway, os_makedirs): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.s3_gateway.object_exists = MagicMock(return_value=True) service.s3_gateway.list_object2 = MagicMock(return_value=self.S3_DIR) service.s3_gateway.download_file = MagicMock(return_value=None) service.copy(self.S3_FOLDER, self.LOCAL_FOLDER, True) os.makedirs.assert_has_calls( [ call("/foo/bar"), call("/foo/baz"), ], any_order=True, ) service.s3_gateway.download_file.assert_has_calls( [ call("bucket", "test_folder/baz/a", "/foo/baz/a"), call("bucket", "test_folder/baz/b", "/foo/baz/b"), ], any_order=True, )
def test_copy_s3_dir_to_local_source_does_not_exist(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.s3_gateway.object_exists = MagicMock(return_value=False) self.assertRaises(ValueError, service.copy, self.S3_FOLDER, self.LOCAL_FOLDER, False)
def test_copy_s3_dir_to_local_recursive_false(self): service = S3StorageService("us-west-1") self.assertRaises(ValueError, service.copy, self.S3_FOLDER, self.LOCAL_FOLDER, False)
def test_copy_local_dir_to_s3_recursive_false(self): service = S3StorageService("us-west-1") with patch("os.path.isdir", return_value=True): self.assertRaises(ValueError, service.copy, self.LOCAL_FOLDER, self.S3_FOLDER, False)
def test_delete_s3(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.delete(self.S3_FILE) service.s3_gateway.delete_object.assert_called_with( "bucket", "test_file")
def test_copy_s3_dir_to_s3_source_and_dest_are_the_same(self): service = S3StorageService("us-west-1") self.assertRaises(ValueError, service.copy, self.S3_FOLDER, self.S3_FOLDER, True)
def test_copy_s3_to_s3(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() service.copy(self.S3_FILE, self.S3_FILE_COPY) service.s3_gateway.copy.assert_called_with("bucket", "test_file", "bucket", "test_file_copy")
def test_copy_local_to_local(self, MockS3Gateway): service = S3StorageService("us-west-1") service.s3_gateway = MockS3Gateway() self.assertRaises(ValueError, service.copy, self.LOCAL_FILE, self.LOCAL_FILE)