def compose_objects(self,
source_resources,
destination_resource,
request_config=None):
"""See CloudApi class for function doc strings."""
if not source_resources:
raise cloud_errors.GcsApiError(
'Compose requires at least one component object.')
if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL:
raise cloud_errors.GcsApiError(
'Compose was called with {} objects. The limit is {}.'.format(
len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL))
validated_request_config = cloud_api.get_provider_request_config(
request_config, GcsRequestConfig)
source_messages = []
for source in source_resources:
source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry(
name=source.storage_url.object_name)
if source.storage_url.generation is not None:
generation = int(source.storage_url.generation)
source_message.generation = generation
source_messages.append(source_message)
compose_request_payload = self.messages.ComposeRequest(
sourceObjects=source_messages,
destination=gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url))
compose_request = self.messages.StorageObjectsComposeRequest(
composeRequest=compose_request_payload,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
ifGenerationMatch=(
validated_request_config.precondition_generation_match),
ifMetagenerationMatch=(
validated_request_config.precondition_metageneration_match),
)
if validated_request_config.predefined_acl_string is not None:
compose_request.destinationPredefinedAcl = getattr(
self.messages.StorageObjectsComposeRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
return gcs_metadata_util.get_object_resource_from_metadata(
self.client.objects.Compose(compose_request))
def upload_object(self,
source_stream,
destination_resource,
progress_callback=None,
request_config=None):
"""See CloudApi class for function doc strings."""
# Doing this as a default argument above can lead to unexpected bugs:
# https://docs.python-guide.org/writing/gotchas/#mutable-default-arguments
if isinstance(request_config, GcsRequestConfig):
validated_request_config = request_config
else:
validated_request_config = cloud_api.convert_to_provider_request_config(
request_config, GcsRequestConfig)
if request_config.size is None:
# Size is required so that apitools_transfer can pick the
# optimal upload strategy.
raise cloud_errors.GcsApiError(
'Upload failed due to missing size. Destination: {}'.format(
destination_resource.storage_url.url_string))
object_metadata = self.messages.Object(
name=destination_resource.storage_url.object_name,
bucket=destination_resource.storage_url.bucket_name,
md5Hash=validated_request_config.md5_hash)
return self._upload_object(
source_stream,
object_metadata,
apitools_strategy=apitools_transfer.SIMPLE_UPLOAD,
progress_callback=progress_callback,
request_config=validated_request_config,
serialization_data=None)
def upload_object(self,
source_stream,
destination_resource,
progress_callback=None,
request_config=None):
"""See CloudApi class for function doc strings."""
validated_request_config = cloud_api.get_provider_request_config(
request_config, GcsRequestConfig)
if request_config.size is None:
# Size is required so that apitools_transfer can pick the
# optimal upload strategy.
raise cloud_errors.GcsApiError(
'Upload failed due to missing size. Destination: {}'.format(
destination_resource.storage_url.url_string))
object_metadata = self.messages.Object(
name=destination_resource.storage_url.object_name,
bucket=destination_resource.storage_url.bucket_name,
md5Hash=validated_request_config.md5_hash)
return self._upload_object(
source_stream,
object_metadata,
apitools_strategy=apitools_transfer.SIMPLE_UPLOAD,
progress_callback=progress_callback,
request_config=validated_request_config,
serialization_data=None)
def list_objects(self,
bucket_name,
prefix=None,
delimiter=None,
all_versions=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsListRequest)
global_params = None
if fields_scope == cloud_api.FieldsScope.SHORT:
global_params = self.messages.StandardQueryParameters()
global_params.fields = (
'prefixes,items/name,items/size,items/generation,nextPageToken'
)
object_list = None
while True:
apitools_request = self.messages.StorageObjectsListRequest(
bucket=bucket_name,
prefix=prefix,
delimiter=delimiter,
versions=all_versions,
projection=projection,
pageToken=object_list.nextPageToken if object_list else None,
maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE)
try:
object_list = self.client.objects.List(
apitools_request, global_params=global_params)
except apitools_exceptions.HttpError as error:
core_exceptions.reraise(cloud_errors.GcsApiError(error))
# Yield objects.
# TODO(b/160238394) Decrypt metadata fields if necessary.
for object_metadata in object_list.items:
object_metadata.bucket = bucket_name
yield gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
# Yield prefixes.
for prefix_string in object_list.prefixes:
yield resource_reference.PrefixResource(storage_url.CloudUrl(
scheme=storage_url.ProviderPrefix.GCS,
bucket_name=bucket_name,
object_name=prefix_string),
prefix=prefix_string)
if not object_list.nextPageToken:
break
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._GetProjection(
fields_scope, self.messages.StorageBucketsListRequest)
request = self.messages.StorageBucketsListRequest(
project=properties.VALUES.core.project.GetOrFail(),
projection=projection)
global_params = None
if fields_scope == cloud_api.FieldsScope.SHORT:
global_params = self.messages.StandardQueryParameters()
global_params.fields = 'items/name'
# TODO(b/160238394) Decrypt metadata fields if necessary.
bucket_iter = list_pager.YieldFromList(
self.client.buckets,
request,
batch_size=cloud_api.NUM_ITEMS_PER_LIST_PAGE,
global_params=global_params)
try:
for bucket in bucket_iter:
yield _bucket_resource_from_metadata(bucket)
except apitools_exceptions.HttpError as error:
core_exceptions.reraise(cloud_errors.GcsApiError(error))
def compose_objects(self,
source_resources,
destination_resource,
request_config,
original_source_resource=None):
"""See CloudApi class for function doc strings."""
if not source_resources:
raise cloud_errors.GcsApiError(
'Compose requires at least one component object.')
if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL:
raise cloud_errors.GcsApiError(
'Compose was called with {} objects. The limit is {}.'.format(
len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL))
source_messages = []
for source in source_resources:
source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry(
name=source.storage_url.object_name)
if source.storage_url.generation is not None:
generation = int(source.storage_url.generation)
source_message.generation = generation
source_messages.append(source_message)
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if original_source_resource and isinstance(
original_source_resource,
resource_reference.FileObjectResource):
original_source_file_path = (
original_source_resource.storage_url.object_name)
else:
original_source_file_path = None
gcs_metadata_util.update_object_metadata_from_request_config(
destination_metadata, request_config, original_source_file_path)
compose_request_payload = self.messages.ComposeRequest(
sourceObjects=source_messages, destination=destination_metadata)
compose_request = self.messages.StorageObjectsComposeRequest(
composeRequest=compose_request_payload,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=request_config.
precondition_metageneration_match)
if request_config.resource_args:
encryption_key = request_config.resource_args.encryption_key
if encryption_key and encryption_key.type == encryption_util.KeyType.CMEK:
compose_request.kmsKeyName = encryption_key.key
if request_config.predefined_acl_string is not None:
compose_request.destinationPredefinedAcl = getattr(
self.messages.StorageObjectsComposeRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
with self._encryption_headers_context(encryption_key):
return gcs_metadata_util.get_object_resource_from_metadata(
self.client.objects.Compose(compose_request))
