def get_object_metadata(self,
bucket_name,
object_name,
request_config=None,
generation=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
if generation:
generation = int(generation)
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsGetRequest)
decryption_key = getattr(
getattr(request_config, 'resource_args', None), 'decryption_key',
None)
with self._encryption_headers_context(decryption_key):
object_metadata = self.client.objects.Get(
self.messages.StorageObjectsGetRequest(bucket=bucket_name,
object=object_name,
generation=generation,
projection=projection))
return gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
def upload_object(self,
source_stream,
destination_resource,
progress_callback=None,
request_config=None,
serialization_data=None,
tracker_callback=None,
upload_strategy=cloud_api.UploadStrategy.SIMPLE):
"""See CloudApi class for function doc strings."""
del progress_callback # Unused.
if self._upload_http_client is None:
self._upload_http_client = transports.GetApitoolsTransport()
validated_request_config = cloud_api.get_provider_request_config(
request_config, GcsRequestConfig)
if upload_strategy == cloud_api.UploadStrategy.SIMPLE:
upload = gcs_upload.SimpleUpload(self, self._upload_http_client,
source_stream,
DEFAULT_CONTENT_TYPE,
destination_resource,
validated_request_config)
elif upload_strategy == cloud_api.UploadStrategy.RESUMABLE:
upload = gcs_upload.ResumableUpload(
self, self._upload_http_client, source_stream,
DEFAULT_CONTENT_TYPE, destination_resource,
validated_request_config, serialization_data, tracker_callback)
else:
raise command_errors.Error('Invalid upload strategy: {}.'.format(
upload_strategy.value))
return gcs_metadata_util.get_object_resource_from_metadata(
upload.run())
def get_object_metadata(self,
bucket_name,
object_name,
generation=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
if generation:
generation = int(generation)
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsGetRequest)
# TODO(b/160238394) Decrypt metadata fields if necessary.
try:
object_metadata = self.client.objects.Get(
self.messages.StorageObjectsGetRequest(bucket=bucket_name,
object=object_name,
generation=generation,
projection=projection))
except apitools_exceptions.HttpNotFoundError:
raise cloud_errors.NotFoundError('Object not found: {}'.format(
storage_url.CloudUrl(storage_url.ProviderPrefix.GCS,
bucket_name, object_name,
generation).url_string))
return gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
def upload_object(self,
source_stream,
destination_resource,
request_config,
source_resource=None,
serialization_data=None,
tracker_callback=None,
upload_strategy=cloud_api.UploadStrategy.SIMPLE):
"""See CloudApi class for function doc strings."""
if self._upload_http_client is None:
self._upload_http_client = transports.GetApitoolsTransport(
redact_request_body_reason=
('Object data is not displayed to keep the log output clean.'
' Set log_http_show_request_body property to True to print the'
' body of this request.'))
if source_resource:
source_path = source_resource.storage_url.versionless_url_string
else:
source_path = None
should_gzip_in_flight = gzip_util.should_gzip_in_flight(
request_config.gzip_settings, source_path)
if should_gzip_in_flight:
log.info('Using compressed transport encoding for {}.'.format(
source_path))
if upload_strategy == cloud_api.UploadStrategy.SIMPLE:
upload = gcs_upload.SimpleUpload(self, self._upload_http_client,
source_stream,
destination_resource,
should_gzip_in_flight,
request_config, source_resource)
elif upload_strategy == cloud_api.UploadStrategy.RESUMABLE:
upload = gcs_upload.ResumableUpload(
self, self._upload_http_client, source_stream,
destination_resource, should_gzip_in_flight, request_config,
source_resource, serialization_data, tracker_callback)
else:
raise command_errors.Error('Invalid upload strategy: {}.'.format(
upload_strategy.value))
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
try:
with self._encryption_headers_context(encryption_key):
metadata = upload.run()
except (
apitools_exceptions.StreamExhausted,
apitools_exceptions.TransferError,
) as error:
raise cloud_errors.ResumableUploadAbortError(
'{}\n This likely occurred because the file being uploaded changed '
'size between resumable upload attempts. If this error persists, try '
'deleting the tracker files present in {}'.format(
str(error),
properties.VALUES.storage.tracker_files_directory.Get()))
return gcs_metadata_util.get_object_resource_from_metadata(metadata)
def list_objects(self,
bucket_name,
prefix=None,
delimiter=None,
all_versions=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsListRequest)
global_params = None
if fields_scope == cloud_api.FieldsScope.SHORT:
global_params = self.messages.StandardQueryParameters()
global_params.fields = (
'prefixes,items/name,items/size,items/generation,nextPageToken'
)
object_list = None
while True:
apitools_request = self.messages.StorageObjectsListRequest(
bucket=bucket_name,
prefix=prefix,
delimiter=delimiter,
versions=all_versions,
projection=projection,
pageToken=object_list.nextPageToken if object_list else None,
maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE)
try:
object_list = self.client.objects.List(
apitools_request, global_params=global_params)
except apitools_exceptions.HttpError as e:
core_exceptions.reraise(
cloud_errors.translate_error(e, _ERROR_TRANSLATION))
# Yield objects.
# TODO(b/160238394) Decrypt metadata fields if necessary.
for object_metadata in object_list.items:
object_metadata.bucket = bucket_name
yield gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
# Yield prefixes.
for prefix_string in object_list.prefixes:
yield resource_reference.PrefixResource(storage_url.CloudUrl(
scheme=storage_url.ProviderPrefix.GCS,
bucket_name=bucket_name,
object_name=prefix_string),
prefix=prefix_string)
if not object_list.nextPageToken:
break
def compose_objects(self,
source_resources,
destination_resource,
request_config=None):
"""See CloudApi class for function doc strings."""
if not source_resources:
raise cloud_errors.GcsApiError(
'Compose requires at least one component object.')
if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL:
raise cloud_errors.GcsApiError(
'Compose was called with {} objects. The limit is {}.'.format(
len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL))
validated_request_config = cloud_api.get_provider_request_config(
request_config, GcsRequestConfig)
source_messages = []
for source in source_resources:
source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry(
name=source.storage_url.object_name)
if source.storage_url.generation is not None:
generation = int(source.storage_url.generation)
source_message.generation = generation
source_messages.append(source_message)
compose_request_payload = self.messages.ComposeRequest(
sourceObjects=source_messages,
destination=gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url))
compose_request = self.messages.StorageObjectsComposeRequest(
composeRequest=compose_request_payload,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
ifGenerationMatch=(
validated_request_config.precondition_generation_match),
ifMetagenerationMatch=(
validated_request_config.precondition_metageneration_match),
)
if validated_request_config.predefined_acl_string is not None:
compose_request.destinationPredefinedAcl = getattr(
self.messages.StorageObjectsComposeRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
return gcs_metadata_util.get_object_resource_from_metadata(
self.client.objects.Compose(compose_request))
def patch_object_metadata(self,
bucket_name,
object_name,
object_resource,
fields_scope=cloud_api.FieldsScope.NO_ACL,
generation=None,
request_config=None):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
if generation:
generation = int(generation)
if not request_config:
request_config = GcsRequestConfig()
predefined_acl = None
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsPatchRequest.
PredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsPatchRequest)
# Assume parameters are only for identifying what needs to be patched, and
# the resource contains the desired patched metadata values.
patched_metadata = object_resource.metadata
if not patched_metadata:
object_resource.metadata = gcs_metadata_util.get_apitools_metadata_from_url(
object_resource.storage_url)
request = self.messages.StorageObjectsPatchRequest(
bucket=bucket_name,
object=object_name,
objectResource=object_resource.metadata,
generation=generation,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=request_config.
precondition_metageneration_match,
predefinedAcl=predefined_acl,
projection=projection)
updated_metadata = self.client.objects.Patch(request)
return gcs_metadata_util.get_object_resource_from_metadata(
updated_metadata)
def get_object_metadata(self,
bucket_name,
object_name,
generation=None,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
if generation:
generation = int(generation)
projection = self._get_projection(
fields_scope, self.messages.StorageObjectsGetRequest)
# TODO(b/160238394) Decrypt metadata fields if necessary.
object_metadata = self.client.objects.Get(
self.messages.StorageObjectsGetRequest(bucket=bucket_name,
object=object_name,
generation=generation,
projection=projection))
return gcs_metadata_util.get_object_resource_from_metadata(
object_metadata)
def copy_object(self,
source_resource,
destination_resource,
progress_callback=None,
request_config=None):
"""See super class."""
# TODO(b/161898251): Implement encryption and decryption.
if not request_config:
request_config = GcsRequestConfig()
destination_metadata = getattr(destination_resource, 'metadata', None)
if not destination_metadata:
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if source_resource.metadata:
gcs_metadata_util.copy_select_object_metadata(
source_resource.metadata, destination_metadata)
if request_config.max_bytes_per_call:
max_bytes_per_call = request_config.max_bytes_per_call
else:
max_bytes_per_call = scaled_integer.ParseInteger(
properties.VALUES.storage.copy_chunk_size.Get())
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsRewriteRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
else:
predefined_acl = None
if source_resource.generation is None:
source_generation = None
else:
source_generation = int(source_resource.generation)
tracker_file_path = tracker_file_util.get_tracker_file_path(
destination_resource.storage_url,
tracker_file_util.TrackerFileType.REWRITE,
source_resource.storage_url)
rewrite_parameters_hash = tracker_file_util.hash_gcs_rewrite_parameters_for_tracker_file(
source_resource,
destination_resource,
destination_metadata,
request_config=request_config)
try:
resume_rewrite_token = tracker_file_util.read_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash)
log.debug('Found rewrite token. Resuming copy.')
except files.MissingFileError:
resume_rewrite_token = None
log.debug('No rewrite token found. Starting copy from scratch.')
while True:
request = self.messages.StorageObjectsRewriteRequest(
sourceBucket=source_resource.storage_url.bucket_name,
sourceObject=source_resource.storage_url.object_name,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
object=destination_metadata,
sourceGeneration=source_generation,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=(
request_config.precondition_metageneration_match),
destinationPredefinedAcl=predefined_acl,
rewriteToken=resume_rewrite_token,
maxBytesRewrittenPerCall=max_bytes_per_call)
rewrite_response = self.client.objects.Rewrite(request)
processed_bytes = rewrite_response.totalBytesRewritten
if progress_callback:
progress_callback(processed_bytes)
if rewrite_response.done:
break
elif not resume_rewrite_token:
resume_rewrite_token = rewrite_response.rewriteToken
tracker_file_util.write_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash,
rewrite_response.rewriteToken)
tracker_file_util.delete_tracker_file(tracker_file_path)
return gcs_metadata_util.get_object_resource_from_metadata(
rewrite_response.resource)
def compose_objects(self,
source_resources,
destination_resource,
request_config,
original_source_resource=None):
"""See CloudApi class for function doc strings."""
if not source_resources:
raise cloud_errors.GcsApiError(
'Compose requires at least one component object.')
if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL:
raise cloud_errors.GcsApiError(
'Compose was called with {} objects. The limit is {}.'.format(
len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL))
source_messages = []
for source in source_resources:
source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry(
name=source.storage_url.object_name)
if source.storage_url.generation is not None:
generation = int(source.storage_url.generation)
source_message.generation = generation
source_messages.append(source_message)
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if original_source_resource and isinstance(
original_source_resource,
resource_reference.FileObjectResource):
original_source_file_path = (
original_source_resource.storage_url.object_name)
else:
original_source_file_path = None
gcs_metadata_util.update_object_metadata_from_request_config(
destination_metadata, request_config, original_source_file_path)
compose_request_payload = self.messages.ComposeRequest(
sourceObjects=source_messages, destination=destination_metadata)
compose_request = self.messages.StorageObjectsComposeRequest(
composeRequest=compose_request_payload,
destinationBucket=destination_resource.storage_url.bucket_name,
destinationObject=destination_resource.storage_url.object_name,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=request_config.
precondition_metageneration_match)
if request_config.resource_args:
encryption_key = request_config.resource_args.encryption_key
if encryption_key and encryption_key.type == encryption_util.KeyType.CMEK:
compose_request.kmsKeyName = encryption_key.key
if request_config.predefined_acl_string is not None:
compose_request.destinationPredefinedAcl = getattr(
self.messages.StorageObjectsComposeRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
with self._encryption_headers_context(encryption_key):
return gcs_metadata_util.get_object_resource_from_metadata(
self.client.objects.Compose(compose_request))
def copy_object(self,
source_resource,
destination_resource,
request_config,
progress_callback=None):
"""See super class."""
destination_metadata = getattr(destination_resource, 'metadata', None)
if not destination_metadata:
destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url(
destination_resource.storage_url)
if source_resource.metadata:
gcs_metadata_util.copy_select_object_metadata(
source_resource.metadata, destination_metadata, request_config)
gcs_metadata_util.update_object_metadata_from_request_config(
destination_metadata, request_config)
if request_config.max_bytes_per_call:
max_bytes_per_call = request_config.max_bytes_per_call
else:
max_bytes_per_call = scaled_integer.ParseInteger(
properties.VALUES.storage.copy_chunk_size.Get())
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsRewriteRequest.
DestinationPredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
else:
predefined_acl = None
if source_resource.generation is None:
source_generation = None
else:
source_generation = int(source_resource.generation)
tracker_file_path = tracker_file_util.get_tracker_file_path(
destination_resource.storage_url,
tracker_file_util.TrackerFileType.REWRITE,
source_url=source_resource.storage_url)
rewrite_parameters_hash = tracker_file_util.hash_gcs_rewrite_parameters_for_tracker_file(
source_resource,
destination_resource,
destination_metadata,
request_config=request_config)
try:
resume_rewrite_token = tracker_file_util.read_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash)
log.debug('Found rewrite token. Resuming copy.')
except files.MissingFileError:
resume_rewrite_token = None
log.debug('No rewrite token found. Starting copy from scratch.')
with self._encryption_headers_for_rewrite_call_context(request_config):
while True:
request = self.messages.StorageObjectsRewriteRequest(
sourceBucket=source_resource.storage_url.bucket_name,
sourceObject=source_resource.storage_url.object_name,
destinationBucket=destination_resource.storage_url.
bucket_name,
destinationObject=destination_resource.storage_url.
object_name,
object=destination_metadata,
sourceGeneration=source_generation,
ifGenerationMatch=copy_util.get_generation_match_value(
request_config),
ifMetagenerationMatch=request_config.
precondition_metageneration_match,
destinationPredefinedAcl=predefined_acl,
rewriteToken=resume_rewrite_token,
maxBytesRewrittenPerCall=max_bytes_per_call)
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
if encryption_key and encryption_key.type == encryption_util.KeyType.CMEK:
# This key is also provided in destination_metadata.kmsKeyName by
# update_object_metadata_from_request_config. This has no effect on
# the copy object request, which references the field below, and is a
# side-effect of logic required for uploads and compose operations.
request.destinationKmsKeyName = encryption_key.key
rewrite_response = self.client.objects.Rewrite(request)
processed_bytes = rewrite_response.totalBytesRewritten
if progress_callback:
progress_callback(processed_bytes)
if rewrite_response.done:
break
if not resume_rewrite_token:
resume_rewrite_token = rewrite_response.rewriteToken
if source_resource.size >= scaled_integer.ParseInteger(
properties.VALUES.storage.resumable_threshold.Get(
)):
tracker_file_util.write_rewrite_tracker_file(
tracker_file_path, rewrite_parameters_hash,
rewrite_response.rewriteToken)
tracker_file_util.delete_tracker_file(tracker_file_path)
return gcs_metadata_util.get_object_resource_from_metadata(
rewrite_response.resource)
def _upload_object(self,
source_stream,
object_metadata,
request_config,
apitools_strategy=apitools_transfer.SIMPLE_UPLOAD,
progress_callback=None,
serialization_data=None,
total_size=0,
tracker_callback=None):
# pylint: disable=g-doc-args
"""GCS-specific upload implementation. Adds args to Cloud API interface.
Additional args:
object_metadata (messages.Object): Apitools metadata for object to
upload.
apitools_strategy (str): SIMPLE_UPLOAD or RESUMABLE_UPLOAD constant in
apitools.base.py.transfer.
serialization_data (str): Implementation-specific JSON string of a dict
containing serialization information for the download.
total_size (int): Total size of the upload in bytes.
If streaming, total size is None.
tracker_callback (function): Callback that keeps track of upload progress.
Returns:
Uploaded object metadata in an ObjectResource.
Raises:
ValueError if an object can't be uploaded with the provided metadata.
"""
predefined_acl = None
if request_config.predefined_acl_string:
predefined_acl = getattr(
self.messages.StorageObjectsInsertRequest.
PredefinedAclValueValuesEnum,
request_config.predefined_acl_string)
# TODO(b/160998052): Use encryption_wrapper to generate encryption headers.
# Fresh upload. Prepare arguments.
if not serialization_data:
content_type = object_metadata.contentType
if not content_type:
content_type = DEFAULT_CONTENT_TYPE
request = self.messages.StorageObjectsInsertRequest(
bucket=object_metadata.bucket,
object=object_metadata,
ifGenerationMatch=request_config.precondition_generation_match,
ifMetagenerationMatch=(
request_config.precondition_metageneration_match),
predefinedAcl=predefined_acl)
if apitools_strategy == apitools_transfer.SIMPLE_UPLOAD:
apitools_upload = apitools_transfer.Upload(
source_stream,
content_type,
auto_transfer=True,
chunksize=properties.VALUES.storage.chunk_size.GetInt(),
gzip_encoded=request_config.gzip_encoded,
num_retries=properties.VALUES.storage.max_retries.GetInt(),
progress_callback=progress_callback,
total_size=request_config.size)
result_object_metadata = self.client.objects.Insert(
request, upload=apitools_upload)
return gcs_metadata_util.get_object_resource_from_metadata(
result_object_metadata)
else:
# TODO(b/160998556): Implement resumable upload.
pass
