def compose_objects(self,
source_resources,
destination_resource,
request_config=None):
"""See CloudApi class for function doc strings."""
if not source_resources:
raise cloud_errors.GcsApiError(
'Compose requires at least one component object.')
if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL:
raise cloud_errors.GcsApiError(
'Compose was called with {} objects. The limit is {}.'.format(
len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL))
validated_request_config = cloud_api.get_provider_request_config(
request_config, GcsRequestConfig)
source_messages = []
for source in source_resources:
source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry(
name=source.storage_url.object_name)
if source.storage_url.generation is not None:
generation = int(source.storage_url.generation)
source_message.generation = generation
source_messages.append(source_message)
compose_request_payload = self.messages.ComposeRequest(
sourceObjects=source_messages,
destination=gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url))
compose_request = self.messages.StorageObjectsComposeRequest(
composeRequest=compose_request_payload,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
ifGenerationMatch=(
validated_request_config.precondition_generation_match),
ifMetagenerationMatch=(
validated_request_config.precondition_metageneration_match),
)
if validated_request_config.predefined_acl_string is not None:
compose_request.destinationPredefinedAcl = getattr(
self.messages.StorageObjectsComposeRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
return gcs_metadata_util.get_object_resource_from_metadata(
self.client.objects.Compose(compose_request))
def patch_object_metadata(self,
bucket_name,
object_name,
object_resource,
fields_scope=cloud_api.FieldsScope.NO_ACL,
generation=None,
request_config=None):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
if generation:
generation = int(generation)
if not request_config:
request_config = GcsRequestConfig()
predefined_acl = None
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsPatchRequest.
PredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsPatchRequest)
# Assume parameters are only for identifying what needs to be patched, and
# the resource contains the desired patched metadata values.
patched_metadata = object_resource.metadata
if not patched_metadata:
object_resource.metadata = gcs_metadata_util.get_apitools_metadata_from_url(
object_resource.storage_url)
request = self.messages.StorageObjectsPatchRequest(
bucket=bucket_name,
object=object_name,
objectResource=object_resource.metadata,
generation=generation,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=request_config.
precondition_metageneration_match,
predefinedAcl=predefined_acl,
projection=projection)
updated_metadata = self.client.objects.Patch(request)
return gcs_metadata_util.get_object_resource_from_metadata(
updated_metadata)
def create_bucket(self,
bucket_resource,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._get_projection(
fields_scope, self.messages.StorageBucketsInsertRequest)
if not bucket_resource.metadata:
bucket_resource.metadata = gcs_metadata_util.get_apitools_metadata_from_url(
bucket_resource.storage_url)
request = self.messages.StorageBucketsInsertRequest(
bucket=bucket_resource.metadata,
project=properties.VALUES.core.project.GetOrFail(),
projection=projection)
created_bucket_metadata = self.client.buckets.Insert(request)
return gcs_metadata_util.get_bucket_resource_from_metadata(
created_bucket_metadata)
def copy_object(self,
source_resource,
destination_resource,
progress_callback=None,
request_config=None):
"""See super class."""
# TODO(b/161898251): Implement encryption and decryption.
if not request_config:
request_config = GcsRequestConfig()
destination_metadata = getattr(destination_resource, 'metadata', None)
if not destination_metadata:
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if source_resource.metadata:
gcs_metadata_util.copy_select_object_metadata(
source_resource.metadata, destination_metadata)
if request_config.max_bytes_per_call:
max_bytes_per_call = request_config.max_bytes_per_call
else:
max_bytes_per_call = scaled_integer.ParseInteger(
properties.VALUES.storage.copy_chunk_size.Get())
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsRewriteRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
else:
predefined_acl = None
if source_resource.generation is None:
source_generation = None
else:
source_generation = int(source_resource.generation)
tracker_file_path = tracker_file_util.get_tracker_file_path(
destination_resource.storage_url,
tracker_file_util.TrackerFileType.REWRITE,
source_resource.storage_url)
rewrite_parameters_hash = tracker_file_util.hash_gcs_rewrite_parameters_for_tracker_file(
source_resource,
destination_resource,
destination_metadata,
request_config=request_config)
try:
resume_rewrite_token = tracker_file_util.read_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash)
log.debug('Found rewrite token. Resuming copy.')
except files.MissingFileError:
resume_rewrite_token = None
log.debug('No rewrite token found. Starting copy from scratch.')
while True:
request = self.messages.StorageObjectsRewriteRequest(
sourceBucket=source_resource.storage_url.bucket_name,
sourceObject=source_resource.storage_url.object_name,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
object=destination_metadata,
sourceGeneration=source_generation,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=(
request_config.precondition_metageneration_match),
destinationPredefinedAcl=predefined_acl,
rewriteToken=resume_rewrite_token,
maxBytesRewrittenPerCall=max_bytes_per_call)
rewrite_response = self.client.objects.Rewrite(request)
processed_bytes = rewrite_response.totalBytesRewritten
if progress_callback:
progress_callback(processed_bytes)
if rewrite_response.done:
break
elif not resume_rewrite_token:
resume_rewrite_token = rewrite_response.rewriteToken
tracker_file_util.write_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash,
rewrite_response.rewriteToken)
tracker_file_util.delete_tracker_file(tracker_file_path)
return gcs_metadata_util.get_object_resource_from_metadata(
rewrite_response.resource)
def compose_objects(self,
source_resources,
destination_resource,
request_config,
original_source_resource=None):
"""See CloudApi class for function doc strings."""
if not source_resources:
raise cloud_errors.GcsApiError(
'Compose requires at least one component object.')
if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL:
raise cloud_errors.GcsApiError(
'Compose was called with {} objects. The limit is {}.'.format(
len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL))
source_messages = []
for source in source_resources:
source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry(
name=source.storage_url.object_name)
if source.storage_url.generation is not None:
generation = int(source.storage_url.generation)
source_message.generation = generation
source_messages.append(source_message)
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if original_source_resource and isinstance(
original_source_resource,
resource_reference.FileObjectResource):
original_source_file_path = (
original_source_resource.storage_url.object_name)
else:
original_source_file_path = None
gcs_metadata_util.update_object_metadata_from_request_config(
destination_metadata, request_config, original_source_file_path)
compose_request_payload = self.messages.ComposeRequest(
sourceObjects=source_messages, destination=destination_metadata)
compose_request = self.messages.StorageObjectsComposeRequest(
composeRequest=compose_request_payload,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=request_config.
precondition_metageneration_match)
if request_config.resource_args:
encryption_key = request_config.resource_args.encryption_key
if encryption_key and encryption_key.type == encryption_util.KeyType.CMEK:
compose_request.kmsKeyName = encryption_key.key
if request_config.predefined_acl_string is not None:
compose_request.destinationPredefinedAcl = getattr(
self.messages.StorageObjectsComposeRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
with self._encryption_headers_context(encryption_key):
return gcs_metadata_util.get_object_resource_from_metadata(
self.client.objects.Compose(compose_request))
def copy_object(self,
source_resource,
destination_resource,
request_config,
progress_callback=None):
"""See super class."""
destination_metadata = getattr(destination_resource, 'metadata', None)
if not destination_metadata:
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if source_resource.metadata:
gcs_metadata_util.copy_select_object_metadata(
source_resource.metadata, destination_metadata, request_config)
gcs_metadata_util.update_object_metadata_from_request_config(
destination_metadata, request_config)
if request_config.max_bytes_per_call:
max_bytes_per_call = request_config.max_bytes_per_call
else:
max_bytes_per_call = scaled_integer.ParseInteger(
properties.VALUES.storage.copy_chunk_size.Get())
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsRewriteRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
else:
predefined_acl = None
if source_resource.generation is None:
source_generation = None
else:
source_generation = int(source_resource.generation)
tracker_file_path = tracker_file_util.get_tracker_file_path(
destination_resource.storage_url,
tracker_file_util.TrackerFileType.REWRITE,
source_url=source_resource.storage_url)
rewrite_parameters_hash = tracker_file_util.hash_gcs_rewrite_parameters_for_tracker_file(
source_resource,
destination_resource,
destination_metadata,
request_config=request_config)
try:
resume_rewrite_token = tracker_file_util.read_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash)
log.debug('Found rewrite token. Resuming copy.')
except files.MissingFileError:
resume_rewrite_token = None
log.debug('No rewrite token found. Starting copy from scratch.')
with self._encryption_headers_for_rewrite_call_context(request_config):
while True:
request = self.messages.StorageObjectsRewriteRequest(
sourceBucket=source_resource.storage_url.bucket_name,
sourceObject=source_resource.storage_url.object_name,
destinationBucket=destination_resource.storage_url.
bucket_name,
destinationObject=destination_resource.storage_url.
object_name,
object=destination_metadata,
sourceGeneration=source_generation,
ifGenerationMatch=copy_util.get_generation_match_value(
request_config),
ifMetagenerationMatch=request_config.
precondition_metageneration_match,
destinationPredefinedAcl=predefined_acl,
rewriteToken=resume_rewrite_token,
maxBytesRewrittenPerCall=max_bytes_per_call)
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
if encryption_key and encryption_key.type == encryption_util.KeyType.CMEK:
# This key is also provided in destination_metadata.kmsKeyName by
# update_object_metadata_from_request_config. This has no effect on
# the copy object request, which references the field below, and is a
# side-effect of logic required for uploads and compose operations.
request.destinationKmsKeyName = encryption_key.key
rewrite_response = self.client.objects.Rewrite(request)
processed_bytes = rewrite_response.totalBytesRewritten
if progress_callback:
progress_callback(processed_bytes)
if rewrite_response.done:
break
if not resume_rewrite_token:
resume_rewrite_token = rewrite_response.rewriteToken
if source_resource.size >= scaled_integer.ParseInteger(
properties.VALUES.storage.resumable_threshold.Get(
)):
tracker_file_util.write_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash,
rewrite_response.rewriteToken)
tracker_file_util.delete_tracker_file(tracker_file_path)
return gcs_metadata_util.get_object_resource_from_metadata(
rewrite_response.resource)
