def test_fails_for_daisy_chaining_cloud_copy(self): source_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://b/o1.txt')) destination_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('s3://b/o2.txt')) task = copy_task_factory.get_copy_task(source_resource, destination_resource) self.assertIsInstance(task, daisy_chain_copy_task.DaisyChainCopyTask)
def test_gets_upload_task_for_local_to_cloud_copy(self): source_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('file://o1.txt')) destination_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://b/o2.txt')) task = copy_task_factory.get_copy_task(source_resource, destination_resource) self.assertIsInstance(task, file_upload_task.FileUploadTask)
def test_gets_intra_cloud_copy_task_for_intra_cloud_copy(self): source_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://b/o1.txt')) destination_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://b/o2.txt')) task = copy_task_factory.get_copy_task(source_resource, destination_resource) self.assertIsInstance(task, intra_cloud_copy_task.IntraCloudCopyTask)
def test_object_resources_non_equal_metadata(self): resource1 = resource_reference.ObjectResource(storage_url.CloudUrl( storage_url.ProviderPrefix.GCS, bucket_name='bucket'), metadata={1: 2}) resource2 = resource_reference.ObjectResource(storage_url.CloudUrl( storage_url.ProviderPrefix.GCS, bucket_name='bucket'), metadata={1: 3}) self.assertNotEqual(resource1, resource2)
def test_fails_for_local_to_local_copy(self): source_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('file://o1.txt')) destination_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('file://o2.txt')) with self.assertRaises(ValueError): copy_task_factory.get_copy_task(source_resource, destination_resource)
def test_object_resources_non_equal_etags(self): resource1 = resource_reference.ObjectResource(storage_url.CloudUrl( storage_url.ProviderPrefix.GCS, bucket_name='bucket'), etag='e1') resource2 = resource_reference.ObjectResource(storage_url.CloudUrl( storage_url.ProviderPrefix.GCS, bucket_name='bucket'), etag='e2') self.assertNotEqual(resource1, resource2)
def test_object_resources_non_equal_generations(self): resource1 = resource_reference.ObjectResource( storage_url.CloudUrl(storage_url.ProviderPrefix.GCS, bucket_name='b', object_name='o', generation='g')) resource2 = resource_reference.ObjectResource( storage_url.CloudUrl(storage_url.ProviderPrefix.GCS, bucket_name='b', object_name='o', generation='g2')) self.assertNotEqual(resource1, resource2)
def test_execute_fails_for_local_to_cloud_copy(self): source_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('file://o.txt'), metadata=self.source_metadata) destination_resource = resource_reference.UnknownResource( storage_url.storage_url_from_string('gs://b/o.txt')) with self.assertRaises(ValueError): daisy_chain_copy_task.DaisyChainCopyTask(source_resource, destination_resource)
def SetUp(self): properties.VALUES.core.project.Set(TEST_PROJECT) self.stdout_seek_position = 0 self.messages = core_apis.GetMessagesModule('storage', 'v1') # bucket1: # object1 # dir1/object2 # dir1/subdir1/object3 # dir1/subdir2/object4 # dir2/subdir3/object5 # bucket2: # dir_object # dir_object/object6 self.bucket1 = test_resources.from_url_string('gs://bucket1') self.bucket2 = test_resources.from_url_string('gs://bucket2') self.object1 = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://bucket1/object1#1'), creation_time=DATETIME, size=0) self.object2 = test_resources.from_url_string( 'gs://bucket1/dir1/object2') self.object3 = test_resources.from_url_string( 'gs://bucket1/dir1/subdir1/object3') self.object4 = test_resources.from_url_string( 'gs://bucket1/dir1/subdir2/object4') self.object5 = test_resources.from_url_string( 'gs://bucket1/dir2/subdir3/object5') self.object6 = test_resources.from_url_string( 'gs://bucket2/dir_object/object6') self.object_duplicate_of_dir = test_resources.from_url_string( 'gs://bucket2/dir_object') self.dir1 = test_resources.from_url_string('gs://bucket1/dir1/') self.dir2 = test_resources.from_url_string('gs://bucket1/dir2/') self.subdir1 = test_resources.from_url_string('gs://bucket1/dir1/subdir1/') self.subdir2 = test_resources.from_url_string('gs://bucket1/dir1/subdir2/') self.subdir3 = test_resources.from_url_string('gs://bucket1/dir2/subdir3/') self.dir_duplicate_of_object = test_resources.from_url_string( 'gs://bucket2/dir_object/') self.bucket_resources = [self.bucket1, self.bucket2] self.bucket1_top_level_resources = [self.object1, self.dir1, self.dir2] self.bucket1_dir1_resources = [self.object2, self.subdir1, self.subdir2] self.bucket1_dir1_subdir1_resources = [self.object3] self.bucket1_dir1_subdir2_resources = [self.object4] self.bucket1_dir2_resources = [self.subdir3] self.bucket1_dir2_subdir3_resources = [self.object5] self.bucket1_all_objects = [self.object1, self.object2, self.object3, self.object4, self.object5] self.bucket2_top_level_resources = [self.object_duplicate_of_dir, self.dir_duplicate_of_object] self.bucket2_dir_object_resources = [self.object6]
def test_execute_downloads_file(self, mock_client, mock_file_writer): source_url = storage_url.storage_url_from_string('gs://b/o1.txt') source_resource = resource_reference.ObjectResource(source_url) destination_resource = resource_reference.FileObjectResource( storage_url.storage_url_from_string('file://o2.txt')) task = file_download_task.FileDownloadTask(source_resource, destination_resource) task.execute() mock_file_writer.assert_called_once_with('o2.txt', create_path=True) mock_stream = mock_file_writer() mock_client.download_object.assert_called_once_with( source_url.bucket_name, source_url.object_name, mock_stream)
def test_execute_uploads_file(self, mock_client, mock_stream): source_resource = resource_reference.FileObjectResource( storage_url.storage_url_from_string('file://o1.txt')) destination_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://b/o2.txt')) task = file_upload_task.FileUploadTask(source_resource, destination_resource) task.execute() mock_stream.assert_called_once_with('o1.txt') # We create a new instance of mock_stream to emulate "with ... as ..." # syntax in the task. However, this means "assert_called_once" must be above # because now mock_stream is called twice. mock_client.upload_object.assert_called_once_with( mock_stream(), destination_resource)
def test_execute_copies_file_between_clouds(self, mock_client): test_stream = io.BytesIO() mock_client.download_object.side_effect = _write_to_stream_side_effect( test_stream) mock_client.upload_object.side_effect = ( self._assert_upload_stream_has_correct_data) with mock.patch.object(io, 'BytesIO') as mock_stream_creator: mock_stream_creator.return_value = test_stream source_resource = resource_reference.ObjectResource( storage_url.storage_url_from_string('gs://b/o.txt'), metadata=self.source_metadata) destination_resource = resource_reference.UnknownResource( storage_url.storage_url_from_string('s3://b/o2.txt')) task = daisy_chain_copy_task.DaisyChainCopyTask( source_resource, destination_resource) task.execute() mock_client.download_object.assert_called_once_with( source_resource.bucket, source_resource.name, test_stream) mock_client.upload_object.assert_called_once_with( test_stream, destination_resource)
def get_object_resource(scheme, bucket, name, generation=None): url = storage_url.CloudUrl(scheme, bucket, name, generation) return resource_reference.ObjectResource(url)
def test_gets_object_resource(self): url_string = 'gs://bucket/object#1' parsed_url = storage_url.storage_url_from_string(url_string) resource = resource_reference.ObjectResource(parsed_url) self.assertEqual(test_resources.from_url_string(url_string), resource)