def compose_objects(self, source_resources, destination_resource, request_config=None): """See CloudApi class for function doc strings.""" if not source_resources: raise cloud_errors.GcsApiError( 'Compose requires at least one component object.') if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL: raise cloud_errors.GcsApiError( 'Compose was called with {} objects. The limit is {}.'.format( len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL)) validated_request_config = cloud_api.get_provider_request_config( request_config, GcsRequestConfig) source_messages = [] for source in source_resources: source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry( name=source.storage_url.object_name) if source.storage_url.generation is not None: generation = int(source.storage_url.generation) source_message.generation = generation source_messages.append(source_message) compose_request_payload = self.messages.ComposeRequest( sourceObjects=source_messages, destination=gcs_metadata_util.get_apitools_metadata_from_url( destination_resource.storage_url)) compose_request = self.messages.StorageObjectsComposeRequest( composeRequest=compose_request_payload, destinationBucket=destination_resource.storage_url.bucket_name, destinationObject=destination_resource.storage_url.object_name, ifGenerationMatch=( validated_request_config.precondition_generation_match), ifMetagenerationMatch=( validated_request_config.precondition_metageneration_match), ) if validated_request_config.predefined_acl_string is not None: compose_request.destinationPredefinedAcl = getattr( self.messages.StorageObjectsComposeRequest. DestinationPredefinedAclValueValuesEnum, request_config.predefined_acl_string) return gcs_metadata_util.get_object_resource_from_metadata( self.client.objects.Compose(compose_request))
def upload_object(self, source_stream, destination_resource, progress_callback=None, request_config=None): """See CloudApi class for function doc strings.""" # Doing this as a default argument above can lead to unexpected bugs: # https://docs.python-guide.org/writing/gotchas/#mutable-default-arguments if isinstance(request_config, GcsRequestConfig): validated_request_config = request_config else: validated_request_config = cloud_api.convert_to_provider_request_config( request_config, GcsRequestConfig) if request_config.size is None: # Size is required so that apitools_transfer can pick the # optimal upload strategy. raise cloud_errors.GcsApiError( 'Upload failed due to missing size. Destination: {}'.format( destination_resource.storage_url.url_string)) object_metadata = self.messages.Object( name=destination_resource.storage_url.object_name, bucket=destination_resource.storage_url.bucket_name, md5Hash=validated_request_config.md5_hash) return self._upload_object( source_stream, object_metadata, apitools_strategy=apitools_transfer.SIMPLE_UPLOAD, progress_callback=progress_callback, request_config=validated_request_config, serialization_data=None)
def upload_object(self, source_stream, destination_resource, progress_callback=None, request_config=None): """See CloudApi class for function doc strings.""" validated_request_config = cloud_api.get_provider_request_config( request_config, GcsRequestConfig) if request_config.size is None: # Size is required so that apitools_transfer can pick the # optimal upload strategy. raise cloud_errors.GcsApiError( 'Upload failed due to missing size. Destination: {}'.format( destination_resource.storage_url.url_string)) object_metadata = self.messages.Object( name=destination_resource.storage_url.object_name, bucket=destination_resource.storage_url.bucket_name, md5Hash=validated_request_config.md5_hash) return self._upload_object( source_stream, object_metadata, apitools_strategy=apitools_transfer.SIMPLE_UPLOAD, progress_callback=progress_callback, request_config=validated_request_config, serialization_data=None)
def list_objects(self, bucket_name, prefix=None, delimiter=None, all_versions=None, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" projection = self._get_projection( fields_scope, self.messages.StorageObjectsListRequest) global_params = None if fields_scope == cloud_api.FieldsScope.SHORT: global_params = self.messages.StandardQueryParameters() global_params.fields = ( 'prefixes,items/name,items/size,items/generation,nextPageToken' ) object_list = None while True: apitools_request = self.messages.StorageObjectsListRequest( bucket=bucket_name, prefix=prefix, delimiter=delimiter, versions=all_versions, projection=projection, pageToken=object_list.nextPageToken if object_list else None, maxResults=cloud_api.NUM_ITEMS_PER_LIST_PAGE) try: object_list = self.client.objects.List( apitools_request, global_params=global_params) except apitools_exceptions.HttpError as error: core_exceptions.reraise(cloud_errors.GcsApiError(error)) # Yield objects. # TODO(b/160238394) Decrypt metadata fields if necessary. for object_metadata in object_list.items: object_metadata.bucket = bucket_name yield gcs_metadata_util.get_object_resource_from_metadata( object_metadata) # Yield prefixes. for prefix_string in object_list.prefixes: yield resource_reference.PrefixResource(storage_url.CloudUrl( scheme=storage_url.ProviderPrefix.GCS, bucket_name=bucket_name, object_name=prefix_string), prefix=prefix_string) if not object_list.nextPageToken: break
def list_buckets(self, fields_scope=cloud_api.FieldsScope.NO_ACL): """See super class.""" projection = self._GetProjection( fields_scope, self.messages.StorageBucketsListRequest) request = self.messages.StorageBucketsListRequest( project=properties.VALUES.core.project.GetOrFail(), projection=projection) global_params = None if fields_scope == cloud_api.FieldsScope.SHORT: global_params = self.messages.StandardQueryParameters() global_params.fields = 'items/name' # TODO(b/160238394) Decrypt metadata fields if necessary. bucket_iter = list_pager.YieldFromList( self.client.buckets, request, batch_size=cloud_api.NUM_ITEMS_PER_LIST_PAGE, global_params=global_params) try: for bucket in bucket_iter: yield _bucket_resource_from_metadata(bucket) except apitools_exceptions.HttpError as error: core_exceptions.reraise(cloud_errors.GcsApiError(error))
def compose_objects(self, source_resources, destination_resource, request_config, original_source_resource=None): """See CloudApi class for function doc strings.""" if not source_resources: raise cloud_errors.GcsApiError( 'Compose requires at least one component object.') if len(source_resources) > MAX_OBJECTS_PER_COMPOSE_CALL: raise cloud_errors.GcsApiError( 'Compose was called with {} objects. The limit is {}.'.format( len(source_resources), MAX_OBJECTS_PER_COMPOSE_CALL)) source_messages = [] for source in source_resources: source_message = self.messages.ComposeRequest.SourceObjectsValueListEntry( name=source.storage_url.object_name) if source.storage_url.generation is not None: generation = int(source.storage_url.generation) source_message.generation = generation source_messages.append(source_message) destination_metadata = gcs_metadata_util.get_apitools_metadata_from_url( destination_resource.storage_url) if original_source_resource and isinstance( original_source_resource, resource_reference.FileObjectResource): original_source_file_path = ( original_source_resource.storage_url.object_name) else: original_source_file_path = None gcs_metadata_util.update_object_metadata_from_request_config( destination_metadata, request_config, original_source_file_path) compose_request_payload = self.messages.ComposeRequest( sourceObjects=source_messages, destination=destination_metadata) compose_request = self.messages.StorageObjectsComposeRequest( composeRequest=compose_request_payload, destinationBucket=destination_resource.storage_url.bucket_name, destinationObject=destination_resource.storage_url.object_name, ifGenerationMatch=request_config.precondition_generation_match, ifMetagenerationMatch=request_config. precondition_metageneration_match) if request_config.resource_args: encryption_key = request_config.resource_args.encryption_key if encryption_key and encryption_key.type == encryption_util.KeyType.CMEK: compose_request.kmsKeyName = encryption_key.key if request_config.predefined_acl_string is not None: compose_request.destinationPredefinedAcl = getattr( self.messages.StorageObjectsComposeRequest. DestinationPredefinedAclValueValuesEnum, request_config.predefined_acl_string) encryption_key = getattr(request_config.resource_args, 'encryption_key', None) with self._encryption_headers_context(encryption_key): return gcs_metadata_util.get_object_resource_from_metadata( self.client.objects.Compose(compose_request))