def upload_object(self,
source_stream,
destination_resource,
progress_callback=None,
request_config=None,
serialization_data=None,
tracker_callback=None,
upload_strategy=cloud_api.UploadStrategy.SIMPLE):
"""See CloudApi class for function doc strings."""
del progress_callback # Unused.
if self._upload_http_client is None:
self._upload_http_client = transports.GetApitoolsTransport()
validated_request_config = cloud_api.get_provider_request_config(
request_config, GcsRequestConfig)
if upload_strategy == cloud_api.UploadStrategy.SIMPLE:
upload = gcs_upload.SimpleUpload(self, self._upload_http_client,
source_stream,
DEFAULT_CONTENT_TYPE,
destination_resource,
validated_request_config)
elif upload_strategy == cloud_api.UploadStrategy.RESUMABLE:
upload = gcs_upload.ResumableUpload(
self, self._upload_http_client, source_stream,
DEFAULT_CONTENT_TYPE, destination_resource,
validated_request_config, serialization_data, tracker_callback)
else:
raise command_errors.Error('Invalid upload strategy: {}.'.format(
upload_strategy.value))
return gcs_metadata_util.get_object_resource_from_metadata(
upload.run())
def GenerateIdToken(service_account_id, audience, include_email=False):
"""Generates an id token for the given service account."""
service_account_ref = resources.REGISTRY.Parse(
service_account_id,
collection='iamcredentials.serviceAccounts',
params={
'projectsId': '-',
'serviceAccountsId': service_account_id
})
http_client = transports.GetApitoolsTransport(
enable_resource_quota=False,
response_encoding=transport.ENCODING,
allow_account_impersonation=False)
# pylint: disable=protected-access
iam_client = apis_internal._GetClientInstance('iamcredentials',
'v1',
http_client=http_client)
response = iam_client.projects_serviceAccounts.GenerateIdToken(
iam_client.MESSAGES_MODULE.
IamcredentialsProjectsServiceAccountsGenerateIdTokenRequest(
name=service_account_ref.RelativeName(),
generateIdTokenRequest=iam_client.MESSAGES_MODULE.
GenerateIdTokenRequest(audience=audience,
includeEmail=include_email)))
return response.token
def ReadObject(self, object_ref):
"""Read a file from the given Cloud Storage bucket.
Args:
object_ref: storage_util.ObjectReference, The object to read from.
Raises:
BadFileException if the file read is not successful.
Returns:
file-like object containing the data read.
"""
data = io.BytesIO()
chunksize = self._GetChunkSize()
download = transfer.Download.FromStream(data, chunksize=chunksize)
download.bytes_http = transports.GetApitoolsTransport(
response_encoding=None)
get_req = self.messages.StorageObjectsGetRequest(
bucket=object_ref.bucket, object=object_ref.object)
log.info('Reading [%s]', object_ref)
try:
self.client.objects.Get(get_req, download=download)
except api_exceptions.HttpError as err:
raise exceptions.BadFileException(
'Could not read [{object_}]. Please retry: {err}'.format(
object_=object_ref, err=http_exc.HttpException(err)))
data.seek(0)
return data
def __init__(self):
self.client = core_apis.GetClientInstance('storage', 'v1')
self.messages = core_apis.GetMessagesModule('storage', 'v1')
self._stream_response_handler = _StorageStreamResponseHandler()
self._download_http_client = transports.GetApitoolsTransport(
response_encoding=None,
response_handler=self._stream_response_handler)
def HttpClient(self):
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
assert self.active
from googlecloudsdk.core.credentials import transports # pylint: disable=g-import-not-at-top
http_client = transports.GetApitoolsTransport(
response_encoding=transport.ENCODING, ca_certs=self.ca_certs)
return http_client
def upload_object(self,
source_stream,
destination_resource,
request_config,
source_resource=None,
serialization_data=None,
tracker_callback=None,
upload_strategy=cloud_api.UploadStrategy.SIMPLE):
"""See CloudApi class for function doc strings."""
if self._upload_http_client is None:
self._upload_http_client = transports.GetApitoolsTransport(
redact_request_body_reason=
('Object data is not displayed to keep the log output clean.'
' Set log_http_show_request_body property to True to print the'
' body of this request.'))
if source_resource:
source_path = source_resource.storage_url.versionless_url_string
else:
source_path = None
should_gzip_in_flight = gzip_util.should_gzip_in_flight(
request_config.gzip_settings, source_path)
if should_gzip_in_flight:
log.info('Using compressed transport encoding for {}.'.format(
source_path))
if upload_strategy == cloud_api.UploadStrategy.SIMPLE:
upload = gcs_upload.SimpleUpload(self, self._upload_http_client,
source_stream,
destination_resource,
should_gzip_in_flight,
request_config, source_resource)
elif upload_strategy == cloud_api.UploadStrategy.RESUMABLE:
upload = gcs_upload.ResumableUpload(
self, self._upload_http_client, source_stream,
destination_resource, should_gzip_in_flight, request_config,
source_resource, serialization_data, tracker_callback)
else:
raise command_errors.Error('Invalid upload strategy: {}.'.format(
upload_strategy.value))
encryption_key = getattr(request_config.resource_args,
'encryption_key', None)
try:
with self._encryption_headers_context(encryption_key):
metadata = upload.run()
except (
apitools_exceptions.StreamExhausted,
apitools_exceptions.TransferError,
) as error:
raise cloud_errors.ResumableUploadAbortError(
'{}\n This likely occurred because the file being uploaded changed '
'size between resumable upload attempts. If this error persists, try '
'deleting the tracker files present in {}'.format(
str(error),
properties.VALUES.storage.tracker_files_directory.Get()))
return gcs_metadata_util.get_object_resource_from_metadata(metadata)
def download_object(self,
bucket_name,
object_name,
download_stream,
compressed_encoding=False,
decryption_wrapper=None,
digesters=None,
download_strategy=cloud_api.DownloadStrategy.ONE_SHOT,
generation=None,
object_size=None,
progress_callback=None,
serialization_data=None,
start_byte=0,
end_byte=None):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
if generation:
generation = int(generation)
if not serialization_data:
# New download.
apitools_download = apitools_transfer.Download.FromStream(
download_stream,
auto_transfer=False,
total_size=object_size,
num_retries=DEFAULT_NUM_RETRIES)
apitools_download.bytes_http = transports.GetApitoolsTransport(
response_encoding=None)
else:
# TODO(b/161437901): Handle resumed download.
pass
# TODO(b/161460749) Handle download retries.
request = self.messages.StorageObjectsGetRequest(
bucket=bucket_name,
object=object_name,
generation=generation)
if download_strategy == cloud_api.DownloadStrategy.ONE_SHOT:
return self._download_object(
bucket_name,
object_name,
download_stream,
apitools_download,
request,
compressed_encoding=compressed_encoding,
decryption_wrapper=decryption_wrapper,
generation=generation,
serialization_data=serialization_data,
start_byte=start_byte,
end_byte=end_byte)
else:
# TODO(b/161437901): Handle resumable download.
pass
def HttpClient(self):
assert self.active
if not self.client_key and self.client_cert and self.client_cert_domain:
raise ValueError(
'Kubeconfig authentication requires a client certificate '
'authentication method.')
if self.client_cert_domain:
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
from googlecloudsdk.core import transports # pylint: disable=g-import-not-at-top
http_client = transports.GetApitoolsTransport(
response_encoding=transport.ENCODING,
ca_certs=self.ca_certs,
client_certificate=self.client_cert,
client_key=self.client_key,
client_cert_domain=self.client_cert_domain)
return http_client
from googlecloudsdk.core.credentials import transports # pylint: disable=g-import-not-at-top
http_client = transports.GetApitoolsTransport(
response_encoding=transport.ENCODING, ca_certs=self.ca_certs)
return http_client
def CopyFileFromGCS(self, source_obj_ref, local_path, overwrite=False):
"""Download a file from the given Cloud Storage bucket.
Args:
source_obj_ref: storage_util.ObjectReference, the path of the file on GCS
to download.
local_path: str, the path of the file to download to. Path must be on the
local filesystem.
overwrite: bool, whether or not to overwrite local_path if it already
exists.
Raises:
BadFileException if the file download is not successful.
"""
chunksize = self._GetChunkSize()
download = transfer.Download.FromFile(local_path,
chunksize=chunksize,
overwrite=overwrite)
download.bytes_http = transports.GetApitoolsTransport(
response_encoding=None)
get_req = self.messages.StorageObjectsGetRequest(
bucket=source_obj_ref.bucket, object=source_obj_ref.object)
gsc_path = '{bucket}/{object_path}'.format(
bucket=source_obj_ref.bucket,
object_path=source_obj_ref.object,
)
log.info('Downloading [{gcs}] to [{local_file}]'.format(
local_file=local_path, gcs=gsc_path))
try:
self.client.objects.Get(get_req, download=download)
# When there's a download, Get() returns None so we Get() again to check
# the file size.
response = self.client.objects.Get(get_req)
except api_exceptions.HttpError as err:
raise exceptions.BadFileException(
'Could not copy [{gcs}] to [{local_file}]. Please retry: {err}'
.format(local_file=local_path,
gcs=gsc_path,
err=http_exc.HttpException(err)))
finally:
# Close the stream to release the file handle so we can check its contents
download.stream.close()
file_size = _GetFileSize(local_path)
if response.size != file_size:
log.debug('Download size: {0} bytes, but expected size is {1} '
'bytes.'.format(file_size, response.size))
raise exceptions.BadFileException(
'Cloud Storage download failure. Downloaded file [{0}] does not '
'match Cloud Storage object. Please retry.'.format(local_path))
def _GetClientInstance(version='v1'):
"""Get a client instance for service usage."""
# pylint:disable=protected-access
# Specifically disable resource quota in all cases for service management.
# We need to use this API to turn on APIs and sometimes the user doesn't have
# this API turned on. We should always use the shared project to do this
# so we can bootstrap users getting the appropriate APIs enabled. If the user
# has explicitly set the quota project, then respect that.
enable_resource_quota = (
properties.VALUES.billing.quota_project.IsExplicitlySet())
http_client = transports.GetApitoolsTransport(
response_encoding=transport.ENCODING,
enable_resource_quota=enable_resource_quota)
return apis_internal._GetClientInstance(
'serviceusage', version, http_client=http_client)
def _GetClientInstance(api_name,
api_version,
no_http=False,
http_client=None,
check_response_func=None):
"""Returns an instance of the API client specified in the args.
Args:
api_name: str, The API name (or the command surface name, if different).
api_version: str, The version of the API.
no_http: bool, True to not create an http object for this client.
http_client: bring your own http client to use.
Incompatible with no_http=True.
check_response_func: error handling callback to give to apitools.
Returns:
base_api.BaseApiClient, An instance of the specified API client.
"""
# pylint: disable=g-import-not-at-top
if no_http:
assert http_client is None
elif http_client is None:
# Normal gcloud authentication
# Import http only when needed, as it depends on credential infrastructure
# which is not needed in all cases.
from googlecloudsdk.core.credentials import transports
http_client = transports.GetApitoolsTransport(
response_encoding=transport.ENCODING)
client_class = _GetClientClass(api_name, api_version)
client_instance = client_class(url=_GetEffectiveApiEndpoint(
api_name, api_version, client_class),
get_credentials=False,
http=http_client)
if check_response_func is not None:
client_instance.check_response_func = check_response_func
api_key = properties.VALUES.core.api_key.Get()
if api_key:
client_instance.AddGlobalParam('key', api_key)
header = 'X-Google-Project-Override'
client_instance.additional_http_headers[header] = 'apikey'
return client_instance
def GenerateAccessToken(service_account_id, scopes):
"""Generates an access token for the given service account."""
service_account_ref = resources.REGISTRY.Parse(
service_account_id,
collection='iamcredentials.serviceAccounts',
params={
'projectsId': '-',
'serviceAccountsId': service_account_id
})
http_client = transports.GetApitoolsTransport(
enable_resource_quota=False,
response_encoding=transport.ENCODING,
allow_account_impersonation=False)
# pylint: disable=protected-access
iam_client = apis_internal._GetClientInstance('iamcredentials',
'v1',
http_client=http_client)
try:
response = iam_client.projects_serviceAccounts.GenerateAccessToken(
iam_client.MESSAGES_MODULE.
IamcredentialsProjectsServiceAccountsGenerateAccessTokenRequest(
name=service_account_ref.RelativeName(),
generateAccessTokenRequest=iam_client.MESSAGES_MODULE.
GenerateAccessTokenRequest(scope=scopes)))
return response
except apitools_exceptions.HttpForbiddenError as e:
raise exceptions.HttpException(
e,
error_format='Error {code} (Forbidden) - failed to impersonate '
'[{service_acc}]. Make sure the account that\'s trying '
'to impersonate it has access to the service account '
'itself and the "roles/iam.serviceAccountTokenCreator" '
'role.'.format(code=e.status_code, service_acc=service_account_id))
except apitools_exceptions.HttpError as e:
raise exceptions.HttpException(e)
def download_object(self,
cloud_resource,
download_stream,
compressed_encoding=False,
decryption_wrapper=None,
digesters=None,
download_strategy=cloud_api.DownloadStrategy.RESUMABLE,
progress_callback=None,
start_byte=0,
end_byte=None):
"""See super class."""
# S3 requires a string, but GCS uses an int for generation.
generation = (int(cloud_resource.generation)
if cloud_resource.generation else None)
if start_byte and download_strategy == cloud_api.DownloadStrategy.RESUMABLE:
# Resuming download.
serialization_data = get_download_serialization_data(
cloud_resource, start_byte)
apitools_download = apitools_transfer.Download.FromData(
download_stream,
serialization_data,
num_retries=properties.VALUES.storage.max_retries.GetInt(),
client=self.client)
else:
# New download.
serialization_data = None
apitools_download = apitools_transfer.Download.FromStream(
download_stream,
auto_transfer=False,
total_size=cloud_resource.size,
num_retries=properties.VALUES.storage.max_retries.GetInt())
self._stream_response_handler.update_destination_info(
stream=download_stream,
digesters=digesters,
processed_bytes=start_byte,
progress_callback=progress_callback)
if self._download_http_client is None:
self._download_http_client = transports.GetApitoolsTransport(
response_encoding=None,
response_handler=self._stream_response_handler)
apitools_download.bytes_http = self._download_http_client
# TODO(b/161460749) Handle download retries.
request = self.messages.StorageObjectsGetRequest(
bucket=cloud_resource.bucket,
object=cloud_resource.name,
generation=generation)
if download_strategy == cloud_api.DownloadStrategy.ONE_SHOT:
return self._download_object(
cloud_resource,
download_stream,
apitools_download,
request,
compressed_encoding=compressed_encoding,
decryption_wrapper=decryption_wrapper,
generation=generation,
serialization_data=serialization_data,
start_byte=start_byte,
end_byte=end_byte)
else:
return self._download_object_resumable(
cloud_resource,
download_stream,
apitools_download,
request,
compressed_encoding=compressed_encoding,
decryption_wrapper=decryption_wrapper,
generation=generation,
serialization_data=serialization_data,
start_byte=start_byte,
end_byte=end_byte)
def download_object(self,
cloud_resource,
download_stream,
request_config,
digesters=None,
do_not_decompress=False,
download_strategy=cloud_api.DownloadStrategy.RESUMABLE,
progress_callback=None,
start_byte=0,
end_byte=None):
"""See super class."""
if request_config.system_posix_data:
if cloud_resource.metadata and cloud_resource.metadata.metadata:
custom_metadata_dict = encoding_helper.MessageToDict(
cloud_resource.metadata.metadata)
else:
custom_metadata_dict = {}
posix_attributes_to_set = (
posix_util.get_posix_attributes_from_custom_metadata_dict(
cloud_resource.storage_url.url_string,
custom_metadata_dict))
if not posix_util.are_file_permissions_valid(
cloud_resource.storage_url.url_string,
request_config.system_posix_data, posix_attributes_to_set):
raise posix_util.SETTING_INVALID_POSIX_ERROR
else:
posix_attributes_to_set = None
if download_util.return_and_report_if_nothing_to_download(
cloud_resource, progress_callback):
return cloud_api.DownloadApiClientReturnValue(
posix_attributes=posix_attributes_to_set,
server_reported_encoding=None)
serialization_data = get_download_serialization_data(
cloud_resource, start_byte)
apitools_download = apitools_transfer.Download.FromData(
download_stream,
serialization_data,
num_retries=properties.VALUES.storage.max_retries.GetInt(),
client=self.client)
self._stream_response_handler.update_destination_info(
stream=download_stream,
digesters=digesters,
processed_bytes=start_byte,
progress_callback=progress_callback)
if self._download_http_client is None:
self._download_http_client = transports.GetApitoolsTransport(
response_encoding=None,
response_handler=self._stream_response_handler)
apitools_download.bytes_http = self._download_http_client
additional_headers = {}
if do_not_decompress:
# TODO(b/161453101): Optimize handling of gzip-encoded downloads.
additional_headers['accept-encoding'] = 'gzip'
decryption_key = getattr(request_config.resource_args,
'decryption_key', None)
additional_headers.update(_get_encryption_headers(decryption_key))
if download_strategy == cloud_api.DownloadStrategy.ONE_SHOT:
server_reported_encoding = gcs_download.launch(
apitools_download,
start_byte=start_byte,
end_byte=end_byte,
additional_headers=additional_headers)
else:
server_reported_encoding = gcs_download.launch_retriable(
download_stream,
apitools_download,
start_byte=start_byte,
end_byte=end_byte,
additional_headers=additional_headers)
return cloud_api.DownloadApiClientReturnValue(
posix_attributes=posix_attributes_to_set,
server_reported_encoding=server_reported_encoding)
