def test_fails_for_daisy_chaining_cloud_copy(self):
source_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://b/o1.txt'))
destination_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('s3://b/o2.txt'))
task = copy_task_factory.get_copy_task(source_resource,
destination_resource)
self.assertIsInstance(task, daisy_chain_copy_task.DaisyChainCopyTask)
def test_gets_upload_task_for_local_to_cloud_copy(self):
source_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('file://o1.txt'))
destination_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://b/o2.txt'))
task = copy_task_factory.get_copy_task(source_resource,
destination_resource)
self.assertIsInstance(task, file_upload_task.FileUploadTask)
def test_gets_intra_cloud_copy_task_for_intra_cloud_copy(self):
source_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://b/o1.txt'))
destination_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://b/o2.txt'))
task = copy_task_factory.get_copy_task(source_resource,
destination_resource)
self.assertIsInstance(task, intra_cloud_copy_task.IntraCloudCopyTask)
def test_object_resources_non_equal_metadata(self):
resource1 = resource_reference.ObjectResource(storage_url.CloudUrl(
storage_url.ProviderPrefix.GCS, bucket_name='bucket'),
metadata={1: 2})
resource2 = resource_reference.ObjectResource(storage_url.CloudUrl(
storage_url.ProviderPrefix.GCS, bucket_name='bucket'),
metadata={1: 3})
self.assertNotEqual(resource1, resource2)
def test_fails_for_local_to_local_copy(self):
source_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('file://o1.txt'))
destination_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('file://o2.txt'))
with self.assertRaises(ValueError):
copy_task_factory.get_copy_task(source_resource,
destination_resource)
def test_object_resources_non_equal_etags(self):
resource1 = resource_reference.ObjectResource(storage_url.CloudUrl(
storage_url.ProviderPrefix.GCS, bucket_name='bucket'),
etag='e1')
resource2 = resource_reference.ObjectResource(storage_url.CloudUrl(
storage_url.ProviderPrefix.GCS, bucket_name='bucket'),
etag='e2')
self.assertNotEqual(resource1, resource2)
def test_object_resources_non_equal_generations(self):
resource1 = resource_reference.ObjectResource(
storage_url.CloudUrl(storage_url.ProviderPrefix.GCS,
bucket_name='b',
object_name='o',
generation='g'))
resource2 = resource_reference.ObjectResource(
storage_url.CloudUrl(storage_url.ProviderPrefix.GCS,
bucket_name='b',
object_name='o',
generation='g2'))
self.assertNotEqual(resource1, resource2)
def test_execute_fails_for_local_to_cloud_copy(self):
source_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('file://o.txt'),
metadata=self.source_metadata)
destination_resource = resource_reference.UnknownResource(
storage_url.storage_url_from_string('gs://b/o.txt'))
with self.assertRaises(ValueError):
daisy_chain_copy_task.DaisyChainCopyTask(source_resource,
destination_resource)
def SetUp(self):
properties.VALUES.core.project.Set(TEST_PROJECT)
self.stdout_seek_position = 0
self.messages = core_apis.GetMessagesModule('storage', 'v1')
# bucket1:
# object1
# dir1/object2
# dir1/subdir1/object3
# dir1/subdir2/object4
# dir2/subdir3/object5
# bucket2:
# dir_object
# dir_object/object6
self.bucket1 = test_resources.from_url_string('gs://bucket1')
self.bucket2 = test_resources.from_url_string('gs://bucket2')
self.object1 = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://bucket1/object1#1'),
creation_time=DATETIME, size=0)
self.object2 = test_resources.from_url_string(
'gs://bucket1/dir1/object2')
self.object3 = test_resources.from_url_string(
'gs://bucket1/dir1/subdir1/object3')
self.object4 = test_resources.from_url_string(
'gs://bucket1/dir1/subdir2/object4')
self.object5 = test_resources.from_url_string(
'gs://bucket1/dir2/subdir3/object5')
self.object6 = test_resources.from_url_string(
'gs://bucket2/dir_object/object6')
self.object_duplicate_of_dir = test_resources.from_url_string(
'gs://bucket2/dir_object')
self.dir1 = test_resources.from_url_string('gs://bucket1/dir1/')
self.dir2 = test_resources.from_url_string('gs://bucket1/dir2/')
self.subdir1 = test_resources.from_url_string('gs://bucket1/dir1/subdir1/')
self.subdir2 = test_resources.from_url_string('gs://bucket1/dir1/subdir2/')
self.subdir3 = test_resources.from_url_string('gs://bucket1/dir2/subdir3/')
self.dir_duplicate_of_object = test_resources.from_url_string(
'gs://bucket2/dir_object/')
self.bucket_resources = [self.bucket1, self.bucket2]
self.bucket1_top_level_resources = [self.object1, self.dir1, self.dir2]
self.bucket1_dir1_resources = [self.object2, self.subdir1, self.subdir2]
self.bucket1_dir1_subdir1_resources = [self.object3]
self.bucket1_dir1_subdir2_resources = [self.object4]
self.bucket1_dir2_resources = [self.subdir3]
self.bucket1_dir2_subdir3_resources = [self.object5]
self.bucket1_all_objects = [self.object1, self.object2, self.object3,
self.object4, self.object5]
self.bucket2_top_level_resources = [self.object_duplicate_of_dir,
self.dir_duplicate_of_object]
self.bucket2_dir_object_resources = [self.object6]
def test_execute_downloads_file(self, mock_client, mock_file_writer):
source_url = storage_url.storage_url_from_string('gs://b/o1.txt')
source_resource = resource_reference.ObjectResource(source_url)
destination_resource = resource_reference.FileObjectResource(
storage_url.storage_url_from_string('file://o2.txt'))
task = file_download_task.FileDownloadTask(source_resource,
destination_resource)
task.execute()
mock_file_writer.assert_called_once_with('o2.txt', create_path=True)
mock_stream = mock_file_writer()
mock_client.download_object.assert_called_once_with(
source_url.bucket_name, source_url.object_name, mock_stream)
def test_execute_uploads_file(self, mock_client, mock_stream):
source_resource = resource_reference.FileObjectResource(
storage_url.storage_url_from_string('file://o1.txt'))
destination_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://b/o2.txt'))
task = file_upload_task.FileUploadTask(source_resource,
destination_resource)
task.execute()
mock_stream.assert_called_once_with('o1.txt')
# We create a new instance of mock_stream to emulate "with ... as ..."
# syntax in the task. However, this means "assert_called_once" must be above
# because now mock_stream is called twice.
mock_client.upload_object.assert_called_once_with(
mock_stream(), destination_resource)
def test_execute_copies_file_between_clouds(self, mock_client):
test_stream = io.BytesIO()
mock_client.download_object.side_effect = _write_to_stream_side_effect(
test_stream)
mock_client.upload_object.side_effect = (
self._assert_upload_stream_has_correct_data)
with mock.patch.object(io, 'BytesIO') as mock_stream_creator:
mock_stream_creator.return_value = test_stream
source_resource = resource_reference.ObjectResource(
storage_url.storage_url_from_string('gs://b/o.txt'),
metadata=self.source_metadata)
destination_resource = resource_reference.UnknownResource(
storage_url.storage_url_from_string('s3://b/o2.txt'))
task = daisy_chain_copy_task.DaisyChainCopyTask(
source_resource, destination_resource)
task.execute()
mock_client.download_object.assert_called_once_with(
source_resource.bucket, source_resource.name, test_stream)
mock_client.upload_object.assert_called_once_with(
test_stream, destination_resource)
def get_object_resource(scheme, bucket, name, generation=None): url = storage_url.CloudUrl(scheme, bucket, name, generation) return resource_reference.ObjectResource(url)
def test_gets_object_resource(self):
url_string = 'gs://bucket/object#1'
parsed_url = storage_url.storage_url_from_string(url_string)
resource = resource_reference.ObjectResource(parsed_url)
self.assertEqual(test_resources.from_url_string(url_string), resource)