def test_copy_s3_to_local(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.s3_gateway.download_file = MagicMock(return_value=None)
service.copy(self.S3_FILE, self.LOCAL_FILE)
service.s3_gateway.download_file.assert_called_with(
"bucket", "test_file", str(self.LOCAL_FILE))
def test_copy_s3_dir_to_s3_ok(self, MockS3Gateway, os_makedirs):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.s3_gateway.object_exists = MagicMock(return_value=True)
service.s3_gateway.list_object2 = MagicMock(return_value=self.S3_DIR)
service.s3_gateway.put_object = MagicMock(return_value=None)
service.s3_gateway.copy = MagicMock(return_value=None)
service.copy(self.S3_FOLDER, self.S3_FOLDER_COPY, True)
service.s3_gateway.put_object.assert_has_calls(
[
call("bucket", "test_folder_copy/bar/", ""),
call("bucket", "test_folder_copy/baz/", ""),
],
any_order=True,
)
service.s3_gateway.copy.assert_has_calls(
[
call("bucket", "test_folder/baz/a", "bucket",
"test_folder_copy/baz/a"),
call("bucket", "test_folder/baz/b", "bucket",
"test_folder_copy/baz/b"),
],
any_order=True,
)
def test_file_exists(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.file_exists(self.S3_FILE)
service.s3_gateway.object_exists.assert_called_with(
"bucket", "test_file")
def _download_executables(
repository_path: str,
package_name: str,
) -> None:
s3_region = S3Path(repository_path).region
team, exe_name = _parse_package_name(package_name)
exe_local_path = DEFAULT_EXE_FOLDER + exe_name
exe_s3_path = repository_path + package_name
storage_svc = S3StorageService(s3_region)
storage_svc.copy(exe_s3_path, exe_local_path)
def setUp(self):
storage_svc = S3StorageService("us-west-1")
self.s3_storage_repo = S3MPCInstanceRepository(storage_svc, TEST_BASE_DIR)
self.mpc_instance = MPCInstance(
instance_id=TEST_INSTANCE_ID,
game_name=TEST_GAME_NAME,
mpc_role=TEST_MPC_ROLE,
num_workers=TEST_NUM_WORKERS,
server_ips=TEST_SERVER_IPS,
status=MPCInstanceStatus.CREATED,
input_args=TEST_INPUT_ARGS,
output_args=TEST_OUTPUT_ARGS,
concurrency_args=TEST_CONCURRENCY_ARGS,
input_directory=TEST_INPUT_DIRECTORY,
output_directory=TEST_OUTPUT_DIRECTORY,
)
def test_copy_local_dir_to_s3_recursive_true(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.s3_gateway.put_object = MagicMock(return_value=None)
service.s3_gateway.upload_file = MagicMock(return_value=None)
with patch("os.path.isdir", return_value=True):
with patch("os.walk", return_value=self.LOCAL_DIR):
service.copy(self.LOCAL_FOLDER, self.S3_FOLDER, True)
service.s3_gateway.put_object.assert_called_with(
"bucket", "test_folder/bar/", "")
service.s3_gateway.upload_file.assert_has_calls(
[
call("/foo/baz/a", "bucket", "test_folder/baz/a"),
call("/foo/baz/b", "bucket", "test_folder/baz/b"),
],
any_order=True,
)
def test_copy_s3_dir_to_local_ok(self, MockS3Gateway, os_makedirs):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.s3_gateway.object_exists = MagicMock(return_value=True)
service.s3_gateway.list_object2 = MagicMock(return_value=self.S3_DIR)
service.s3_gateway.download_file = MagicMock(return_value=None)
service.copy(self.S3_FOLDER, self.LOCAL_FOLDER, True)
os.makedirs.assert_has_calls(
[
call("/foo/bar"),
call("/foo/baz"),
],
any_order=True,
)
service.s3_gateway.download_file.assert_has_calls(
[
call("bucket", "test_folder/baz/a", "/foo/baz/a"),
call("bucket", "test_folder/baz/b", "/foo/baz/b"),
],
any_order=True,
)
def test_copy_s3_dir_to_local_source_does_not_exist(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.s3_gateway.object_exists = MagicMock(return_value=False)
self.assertRaises(ValueError, service.copy, self.S3_FOLDER,
self.LOCAL_FOLDER, False)
def test_copy_s3_dir_to_local_recursive_false(self):
service = S3StorageService("us-west-1")
self.assertRaises(ValueError, service.copy, self.S3_FOLDER,
self.LOCAL_FOLDER, False)
def test_copy_local_dir_to_s3_recursive_false(self):
service = S3StorageService("us-west-1")
with patch("os.path.isdir", return_value=True):
self.assertRaises(ValueError, service.copy, self.LOCAL_FOLDER,
self.S3_FOLDER, False)
def test_delete_s3(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.delete(self.S3_FILE)
service.s3_gateway.delete_object.assert_called_with(
"bucket", "test_file")
def test_copy_s3_dir_to_s3_source_and_dest_are_the_same(self):
service = S3StorageService("us-west-1")
self.assertRaises(ValueError, service.copy, self.S3_FOLDER,
self.S3_FOLDER, True)
def test_copy_s3_to_s3(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
service.copy(self.S3_FILE, self.S3_FILE_COPY)
service.s3_gateway.copy.assert_called_with("bucket", "test_file",
"bucket", "test_file_copy")
def test_copy_local_to_local(self, MockS3Gateway):
service = S3StorageService("us-west-1")
service.s3_gateway = MockS3Gateway()
self.assertRaises(ValueError, service.copy, self.LOCAL_FILE,
self.LOCAL_FILE)
