def execute(self, task_status_queue=None):
"""Performs upload."""
progress_callback = progress_callbacks.FilesAndBytesProgressCallback(
status_queue=task_status_queue,
size=self._length,
source_url=self._source_resource.storage_url,
destination_url=self._destination_resource.storage_url,
component_number=self._component_number,
total_components=self._total_components,
operation_name=task_status.OperationName.UPLOADING,
process_id=os.getpid(),
thread_id=threading.get_ident(),
)
source_stream = files.BinaryFileReader(
self._source_resource.storage_url.object_name)
provider = self._destination_resource.storage_url.scheme
with file_part.FilePart(source_stream, self._offset,
self._length) as upload_stream:
api_factory.get_api(provider).upload_object(
upload_stream,
self._destination_resource,
request_config=cloud_api.RequestConfig(
md5_hash=self._source_resource.md5_hash,
size=self._length),
progress_callback=progress_callback)
def execute(self, task_status_queue=None):
log.status.Print('Updating {}...'.format(self._bucket_resource))
provider = self._bucket_resource.storage_url.scheme
request_config = request_config_factory.get_request_config(
self._bucket_resource.storage_url,
user_request_args=self._user_request_args)
try:
api_factory.get_api(provider).patch_bucket(
self._bucket_resource, request_config=request_config)
except errors.GcsApiError as e:
# Service agent does not have the encrypter/decrypter role.
if (e.payload.status_code == 403
and request_config.resource_args.default_encryption_key):
service_agent = api_factory.get_api(
provider).get_service_agent()
requests.AddCryptoKeyPermission(
request_config.resource_args.default_encryption_key,
'serviceAccount:' + service_agent)
api_factory.get_api(provider).patch_bucket(
self._bucket_resource, request_config=request_config)
else:
raise
if task_status_queue:
progress_callbacks.increment_count_callback(task_status_queue)
def _perform_download(self, digesters, progress_callback,
download_strategy, start_byte, end_byte):
"""Prepares file stream, calls API, and validates hash."""
mode = (files.BinaryFileWriterMode.MODIFY
if start_byte else files.BinaryFileWriterMode.TRUNCATE)
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True,
mode=mode) as download_stream:
download_stream.seek(start_byte)
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
self._source_resource,
download_stream,
digesters=digesters,
download_strategy=download_strategy,
progress_callback=progress_callback,
start_byte=start_byte,
end_byte=end_byte)
# TODO(b/172048376): Add crc32c, and make this a loop.
if util.HashAlgorithms.MD5 in digesters:
calculated_digest = util.get_base64_hash_digest_string(
digesters[util.HashAlgorithms.MD5])
util.validate_object_hashes_match(
self._source_resource.storage_url,
self._source_resource.md5_hash, calculated_digest)
def execute(self, task_status_queue=None):
"""Runs download to stream."""
progress_callback = progress_callbacks.FilesAndBytesProgressCallback(
status_queue=task_status_queue,
offset=0,
length=self._source_resource.size,
source_url=self._source_resource.storage_url,
destination_url=self._download_stream.name,
operation_name=task_status.OperationName.DOWNLOADING,
process_id=os.getpid(),
thread_id=threading.get_ident(),
)
request_config = request_config_factory.get_request_config(
self._source_resource.storage_url,
decryption_key_hash=self._source_resource.decryption_key_hash,
user_request_args=self._user_request_args,
)
provider = self._source_resource.storage_url.scheme
api_factory.get_api(provider).download_object(
self._source_resource,
self._download_stream,
request_config,
download_strategy=cloud_api.DownloadStrategy.ONE_SHOT,
progress_callback=progress_callback)
if self._print_created_message:
log.status.Print('Created: {}'.format(self._download_stream.name))
def execute(self, task_status_queue=None):
log.status.Print('Creating {}...'.format(self._bucket_resource))
provider = self._bucket_resource.storage_url.scheme
request_config = request_config_factory.get_request_config(
self._bucket_resource.storage_url,
user_request_args=self._user_request_args)
api_factory.get_api(provider).create_bucket(
self._bucket_resource, request_config=request_config)
def execute(self, callback=None):
destination_url = self._destination_resource.storage_url
provider = destination_url.scheme
with files.BinaryFileReader(self._source_resource.storage_url.
object_name) as upload_stream:
# TODO(b/162069479): Support all of upload_object's parameters.
api_factory.get_api(provider).upload_object(
upload_stream, self._destination_resource)
def execute(self, callback=None):
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True) as download_stream:
provider = self._source_resource.storage_url.scheme
bucket_name = self._source_resource.storage_url.bucket_name
object_name = self._source_resource.storage_url.object_name
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
bucket_name, object_name, download_stream)
def execute(self, task_status_queue=None):
provider = self._object_url.scheme
request_config = request_config_factory.get_request_config(
self._object_url, user_request_args=self._user_request_args)
if self._verbose:
log.status.Print('Removing {}...'.format(self._object_url))
api_factory.get_api(provider).delete_object(self._object_url,
request_config)
if task_status_queue:
progress_callbacks.increment_count_callback(task_status_queue)
def execute(self, callback=None):
destination_url = self._destination_resource.storage_url
provider = destination_url.scheme
source_stream = files.BinaryFileReader(
self._source_resource.storage_url.object_name)
with file_part.FilePart(source_stream, self._offset,
self._length) as upload_stream:
api_factory.get_api(provider).upload_object(
upload_stream,
self._destination_resource,
request_config=cloud_api.RequestConfig(
md5_hash=self._source_resource.md5_hash,
size=self._length))
def __init__(self,
url,
all_versions=False,
error_on_missing_key=True,
fields_scope=cloud_api.FieldsScope.NO_ACL,
get_bucket_metadata=False):
"""Instantiates an iterator that matches the wildcard URL.
Args:
url (CloudUrl): CloudUrl that may contain wildcard that needs expansion.
all_versions (bool): If true, the iterator yields all versions of objects
matching the wildcard. If false, yields just the live object version.
error_on_missing_key (bool): If true, and the encryption key needed to
decrypt an object is missing, the iterator raises an error for that
object.
fields_scope (cloud_api.FieldsScope): Determines amount of metadata
returned by API.
get_bucket_metadata (bool): If true, perform a bucket GET request when
fetching bucket resources
"""
super(CloudWildcardIterator, self).__init__()
url = _compress_url_wildcards(url)
self._url = url
self._all_versions = all_versions
self._error_on_missing_key = error_on_missing_key
self._fields_scope = fields_scope
self._get_bucket_metadata = get_bucket_metadata
self._client = api_factory.get_api(url.scheme)
if url.url_string.endswith(url.delimiter):
# Forces the API to return prefixes instead of their contents.
url = storage_url.storage_url_from_string(
storage_url.rstrip_one_delimiter(url.url_string))
def execute(self, task_status_queue=None):
"""Performs a simple upload. See base class for information on args."""
api = api_factory.get_api(
self._destination_resource.storage_url.scheme)
request_config = request_config_factory.get_request_config(
self._destination_resource.storage_url,
content_type=upload_util.get_content_type(
self._source_resource.storage_url.object_name,
self._source_resource.storage_url.is_pipe),
md5_hash=self._source_resource.md5_hash,
size=self._length)
digesters = upload_util.get_digesters(self._source_resource,
self._destination_resource)
source_stream = upload_util.get_stream(
self._source_resource,
length=self._length,
digesters=digesters,
task_status_queue=task_status_queue,
destination_resource=self._destination_resource)
with source_stream:
uploaded_object_resource = api.upload_object(
source_stream,
self._destination_resource,
request_config,
source_resource=self._source_resource,
upload_strategy=cloud_api.UploadStrategy.SIMPLE)
upload_util.validate_uploaded_object(digesters,
uploaded_object_resource,
task_status_queue)
def execute(self, task_status_queue=None):
source_filename = self._source_resource.storage_url.object_name
size = os.path.getsize(source_filename)
should_perform_single_transfer = (
size < self._composite_upload_threshold or
not self._composite_upload_threshold
)
if should_perform_single_transfer:
file_part_upload_task.FilePartUploadTask(
self._source_resource,
self._destination_resource,
offset=0,
length=size).execute(task_status_queue)
else:
destination_url = self._destination_resource.storage_url
provider = destination_url.scheme
api_instance = api_factory.get_api(provider)
component_count = _get_component_count(
size, api_instance.MAX_OBJECTS_PER_COMPOSE_CALL)
component_size = math.ceil(size / component_count)
file_part_upload_tasks = []
compose_objects_sources = []
delete_object_tasks = []
for component_index in range(component_count):
temporary_component_resource = temporary_components.get_resource(
self._source_resource,
self._destination_resource,
component_index)
compose_objects_sources.append(temporary_component_resource)
offset = component_index * component_size
length = min(component_size, size - offset)
upload_task = file_part_upload_task.FilePartUploadTask(
self._source_resource,
temporary_component_resource,
offset,
length,
component_number=component_index,
total_components=component_count)
file_part_upload_tasks.append(upload_task)
delete_task = delete_object_task.DeleteObjectTask(
temporary_component_resource)
delete_object_tasks.append(delete_task)
compose_objects_tasks = [compose_objects_task.ComposeObjectsTask(
compose_objects_sources, self._destination_resource)]
return [
file_part_upload_tasks,
compose_objects_tasks,
delete_object_tasks,
]
def execute(self, callback=None):
"""Copies file by downloading and uploading in parallel."""
# TODO (b/168712813): Add option to use the Data Transfer component.
daisy_chain_stream = QueuingStream(self._source_resource.size)
# Perform download in a separate thread so that upload can be performed
# simultaneously.
download_thread = threading.Thread(
target=self._run_download, args=(daisy_chain_stream,))
download_thread.start()
destination_client = api_factory.get_api(
self._destination_resource.storage_url.scheme)
request_config = cloud_api.RequestConfig(size=self._source_resource.size)
try:
destination_client.upload_object(
daisy_chain_stream.readable_stream,
self._destination_resource,
request_config=request_config)
except _AbruptShutdownError:
# Not raising daisy_chain_stream.exception_raised here because we want
# to wait for the download thread to finish.
pass
except Exception as e: # pylint: disable=broad-except
# For all the other errors raised during upload, we want to to make
# sure that the download thread is terminated before we re-reaise.
# Hence we catch any exception and store it to be re-raised later.
daisy_chain_stream.shutdown(e)
download_thread.join()
if daisy_chain_stream.exception_raised:
raise daisy_chain_stream.exception_raised
def execute(self, task_status_queue=None):
progress_callback = progress_callbacks.FilesAndBytesProgressCallback(
status_queue=task_status_queue,
size=self._source_resource.size,
source_url=self._source_resource.storage_url,
destination_url=self._destination_resource.storage_url,
operation_name=task_status.OperationName.INTRA_CLOUD_COPYING,
process_id=os.getpid(),
thread_id=threading.get_ident(),
)
# TODO(b/161900052): Support all of copy_object's parameters
provider = self._source_resource.storage_url.scheme
api_factory.get_api(provider).copy_object(
self._source_resource,
self._destination_resource,
progress_callback=progress_callback)
def execute(self, callback=None):
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True) as download_stream:
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
self._source_resource, download_stream)
with files.BinaryFileReader(self._destination_resource.storage_url.
object_name) as completed_download_stream:
downloaded_file_hash = util.get_hash_digest_from_file_stream(
completed_download_stream, util.HashAlgorithms.MD5)
util.validate_object_hashes_match(
self._source_resource.storage_url,
self._source_resource.md5_hash, downloaded_file_hash)
def execute(self, task_status_queue=None):
api_client = api_factory.get_api(
self._source_resource.storage_url.scheme)
if copy_util.check_for_cloud_clobber(self._user_request_args,
api_client,
self._destination_resource):
log.status.Print(
copy_util.get_no_clobber_message(
self._destination_resource.storage_url))
if self._send_manifest_messages:
manifest_util.send_skip_message(
task_status_queue, self._source_resource,
self._destination_resource,
copy_util.get_no_clobber_message(
self._destination_resource.storage_url))
return
progress_callback = progress_callbacks.FilesAndBytesProgressCallback(
status_queue=task_status_queue,
offset=0,
length=self._source_resource.size,
source_url=self._source_resource.storage_url,
destination_url=self._destination_resource.storage_url,
operation_name=task_status.OperationName.INTRA_CLOUD_COPYING,
process_id=os.getpid(),
thread_id=threading.get_ident(),
)
request_config = request_config_factory.get_request_config(
self._destination_resource.storage_url,
decryption_key_hash=self._source_resource.decryption_key_hash,
user_request_args=self._user_request_args)
# TODO(b/161900052): Support all of copy_object's parameters
result_resource = api_client.copy_object(
self._source_resource,
self._destination_resource,
request_config,
progress_callback=progress_callback)
if self._print_created_message:
log.status.Print('Created: {}'.format(result_resource.storage_url))
if self._send_manifest_messages:
manifest_util.send_success_message(
task_status_queue,
self._source_resource,
self._destination_resource,
md5_hash=result_resource.md5_hash)
if self._delete_source:
return task.Output(additional_task_iterators=[[
delete_object_task.DeleteObjectTask(
self._source_resource.storage_url)
]],
messages=None)
def Run(self, args):
if wildcard_iterator.contains_wildcard(args.url):
raise errors.InvalidUrlError(
'Describe does not accept wildcards because it returns a single'
' resource. Please use the `ls` or `buckets list` command for'
' retrieving multiple resources.')
url = storage_url.storage_url_from_string(args.url)
bucket_resource = api_factory.get_api(url.scheme).get_bucket(
url.bucket_name, fields_scope=cloud_api.FieldsScope.FULL)
# MakeSerializable will omit all the None values.
return resource_projector.MakeSerializable(
bucket_resource.get_displayable_bucket_data())
def Run(self, args):
api = api_factory.get_api(storage_url.ProviderPrefix.GCS)
service_agent = api.get_service_agent()
if args.authorize_cmek:
requests.AddCryptoKeyPermission(args.authorize_cmek,
'serviceAccount:' + service_agent)
log.Print(
'Authorized project {} to encrypt and decrypt with key:\n{}'.
format(properties.VALUES.core.project.Get(),
args.authorize_cmek))
else:
log.Print(service_agent)
def execute(self, callback=None):
if self._source_resource.md5_hash:
digesters = {util.HashAlgorithms.MD5: util.get_md5_hash()}
else:
digesters = {}
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True) as download_stream:
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
self._source_resource, download_stream, digesters=digesters)
# TODO(b/172048376): Add crc32c, and make this a loop.
if util.HashAlgorithms.MD5 in digesters:
calculated_digest = util.get_base64_hash_digest_string(
digesters[util.HashAlgorithms.MD5])
util.validate_object_hashes_match(
self._source_resource.storage_url,
self._source_resource.md5_hash, calculated_digest)
def _run_download(self, daisy_chain_stream):
"""Performs the download operation."""
client = api_factory.get_api(self._source_resource.storage_url.scheme)
try:
client.download_object(self._source_resource,
daisy_chain_stream.writable_stream)
except _AbruptShutdownError:
# Shutdown caused by interuption from another thread.
pass
except Exception as e: # pylint: disable=broad-except
# The stack trace of the exception raised in the thread is not visible
# in the caller thread. Hence we catch any exception so that we can
# re-raise them from the parent thread.
daisy_chain_stream.shutdown(e)
def execute(self, task_status_queue=None):
del task_status_queue # Unused.
request_config = request_config_factory.get_request_config(
self._destination_resource.storage_url,
user_request_args=self._user_request_args)
provider = self._destination_resource.storage_url.scheme
created_resource = api_factory.get_api(provider).compose_objects(
self._source_resources,
self._destination_resource,
request_config,
original_source_resource=self._original_source_resource)
return task.Output(messages=[
task.Message(topic=task.Topic.CREATED_RESOURCE,
payload=created_resource),
],
additional_task_iterators=[])
def execute(self, callback=None):
"""Copies file by downloading and uploading."""
# TODO (b/168712813): Rewrite to use Data Transfer component.
print(
'WARNING: The daisy-chaining copy holds all items being copied in'
' memory. This may crash your system if the size of your copy is'
' greater than available memory. Consider using Data Transfer:'
' https://cloud.google.com/products/data-transfer')
provider = self._source_resource.storage_url.scheme
client = api_factory.get_api(provider)
# TODO(b/168489606): Buffer transfer to avoid holding entire download in
# memory at once.
with io.BytesIO() as daisy_chain_stream:
client.download_object(self._source_resource, daisy_chain_stream)
client.upload_object(daisy_chain_stream,
self._destination_resource)
def execute(self, task_status_queue=None):
log.status.Print('Removing {}...'.format(self._url))
api_client = api_factory.get_api(self._url.scheme)
request_config = request_config_factory.get_request_config(self._url)
try:
api_client.delete_bucket(self._url.bucket_name, request_config)
if task_status_queue:
progress_callbacks.increment_count_callback(task_status_queue)
# pylint:disable=broad-except
except Exception as e:
# pylint:enable=broad-except
if 'not empty' in str(e):
raise type(
e
)('Bucket is not empty. To delete all objects and then delete'
' bucket, use: gcloud storage rm -r')
else:
raise
def _perform_download(self, request_config, progress_callback,
download_strategy, start_byte, end_byte, write_mode,
digesters):
"""Prepares file stream, calls API, and validates hash."""
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True,
mode=write_mode) as download_stream:
download_stream.seek(start_byte)
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_download_result = api_factory.get_api(
provider).download_object(
self._source_resource,
download_stream,
request_config,
digesters=digesters,
do_not_decompress=self._do_not_decompress,
download_strategy=download_strategy,
progress_callback=progress_callback,
start_byte=start_byte,
end_byte=end_byte)
if hash_util.HashAlgorithm.MD5 in digesters:
calculated_digest = hash_util.get_base64_hash_digest_string(
digesters[hash_util.HashAlgorithm.MD5])
download_util.validate_download_hash_and_delete_corrupt_files(
self._destination_resource.storage_url.object_name,
self._source_resource.md5_hash, calculated_digest)
# Only for one-shot composite object downloads as final CRC32C validated in
# FinalizeSlicedDownloadTask.
elif (hash_util.HashAlgorithm.CRC32C in digesters
and self._component_number is None):
calculated_digest = crc32c.get_hash(
digesters[hash_util.HashAlgorithm.CRC32C])
download_util.validate_download_hash_and_delete_corrupt_files(
self._destination_resource.storage_url.object_name,
self._source_resource.crc32c_hash, calculated_digest)
return api_download_result
def _run_download(self, start_byte):
"""Performs the download operation."""
request_config = request_config_factory.get_request_config(
self._source_resource.storage_url,
user_request_args=self._user_request_args)
client = api_factory.get_api(self._source_resource.storage_url.scheme)
try:
if self._source_resource.size != 0:
client.download_object(
self._source_resource,
self.writable_stream,
request_config,
start_byte=start_byte,
download_strategy=cloud_api.DownloadStrategy.ONE_SHOT)
except _AbruptShutdownError:
# Shutdown caused by interruption from another thread.
pass
except Exception as e: # pylint: disable=broad-except
# The stack trace of the exception raised in the thread is not visible
# in the caller thread. Hence we catch any exception so that we can
# re-raise them from the parent thread.
self.shutdown(e)
def __init__(self,
url,
all_versions=False,
fields_scope=cloud_api.FieldsScope.NO_ACL):
"""Instantiates an iterator that matches the wildcard URL.
Args:
url (CloudUrl): CloudUrl that may contain wildcard that needs expansion.
all_versions (bool): If true, the iterator yields all versions of objects
matching the wildcard. If false, yields just the live object version.
fields_scope (cloud_api.FieldsScope): Determines amount of metadata
returned by API.
"""
super(CloudWildcardIterator, self).__init__()
url = _compress_url_wildcards(url)
self._url = url
self._all_versions = all_versions
self._fields_scope = fields_scope
self._client = api_factory.get_api(url.scheme)
if url.url_string.endswith(url.delimiter):
# Forces the API to return prefixes instead of their contents.
url = storage_url.storage_url_from_string(
storage_url.rstrip_one_delimiter(url.url_string))
def execute(self, callback=None):
# TODO(b/161900052): Support all of copy_object's parameters
provider = self._source_resource.storage_url.scheme
api_factory.get_api(provider).copy_object(self._source_resource,
self._destination_resource)
def execute(self, task_status_queue=None):
"""Performs upload."""
digesters = self._get_digesters()
destination_url = self._destination_resource.storage_url
provider = destination_url.scheme
api = api_factory.get_api(provider)
request_config = request_config_factory.get_request_config(
destination_url,
content_type=upload_util.get_content_type(
self._source_path, self._source_resource.storage_url.is_pipe),
md5_hash=self._source_resource.md5_hash,
size=self._length,
user_request_args=self._user_request_args)
if self._component_number is None:
source_resource_for_metadata = self._source_resource
else:
source_resource_for_metadata = None
with self._get_upload_stream(digesters,
task_status_queue) as source_stream:
upload_strategy = upload_util.get_upload_strategy(
api, self._length)
if upload_strategy == cloud_api.UploadStrategy.RESUMABLE:
tracker_file_path = tracker_file_util.get_tracker_file_path(
self._destination_resource.storage_url,
tracker_file_util.TrackerFileType.UPLOAD,
component_number=self._component_number)
encryption_key = encryption_util.get_encryption_key()
if encryption_key:
encryption_key_hash = encryption_key.sha256
else:
encryption_key_hash = None
complete = False
tracker_callback = functools.partial(
tracker_file_util.write_resumable_upload_tracker_file,
tracker_file_path, complete, encryption_key_hash)
tracker_data = tracker_file_util.read_resumable_upload_tracker_file(
tracker_file_path)
if (tracker_data is None or tracker_data.encryption_key_sha256
!= encryption_key_hash):
serialization_data = None
else:
# TODO(b/190093425): Print a better message for component uploads once
# the final destination resource is available in ComponentUploadTask.
log.status.Print('Resuming upload for ' +
destination_url.object_name)
serialization_data = tracker_data.serialization_data
if tracker_data.complete:
try:
metadata_request_config = request_config_factory.get_request_config(
destination_url,
decryption_key_hash=encryption_key_hash)
# Providing a decryption key means the response will include the
# object's hash if the keys match, and raise an error if they do
# not. This is desirable since we want to re-upload objects with
# the wrong key, and need the object's hash for validation.
destination_resource = api.get_object_metadata(
destination_url.bucket_name,
destination_url.object_name,
metadata_request_config)
except api_errors.CloudApiError:
# Any problem fetching existing object metadata can be ignored,
# since we'll just re-upload the object.
pass
else:
# The API call will not error if we provide an encryption key but
# the destination is unencrypted, hence the additional (defensive)
# check below.
destination_key_hash = destination_resource.decryption_key_hash
if (destination_key_hash == encryption_key_hash
and self._existing_destination_is_valid(
destination_resource)):
return self._get_output(destination_resource)
attempt_upload = functools.partial(
api.upload_object,
source_stream,
self._destination_resource,
request_config,
source_resource=source_resource_for_metadata,
serialization_data=serialization_data,
tracker_callback=tracker_callback,
upload_strategy=upload_strategy)
def _handle_resumable_upload_error(exc_type, exc_value,
exc_traceback, state):
"""Returns true if resumable upload should retry on error argument."""
del exc_traceback # Unused.
if not (exc_type is api_errors.NotFoundError
or getattr(exc_value, 'status_code', None) == 410):
if exc_type is api_errors.ResumableUploadAbortError:
tracker_file_util.delete_tracker_file(
tracker_file_path)
# Otherwise the error is probably a persistent network issue
# that is already retried by API clients, so we'll keep the tracker
# file to allow the user to retry the upload in a separate run.
return False
tracker_file_util.delete_tracker_file(tracker_file_path)
if state.retrial == 0:
# Ping bucket to see if it exists.
try:
api.get_bucket(self._destination_resource.
storage_url.bucket_name)
except api_errors.CloudApiError as e:
# The user may not have permission to view the bucket metadata,
# so the ping may still be valid for access denied errors.
status = getattr(e, 'status_code', None)
if status not in (401, 403):
raise
return True
# Convert seconds to miliseconds by multiplying by 1000.
destination_resource = retry.Retryer(
max_retrials=properties.VALUES.storage.max_retries.GetInt(
),
wait_ceiling_ms=properties.VALUES.storage.max_retry_delay.
GetInt() * 1000,
exponential_sleep_multiplier=(
properties.VALUES.storage.exponential_sleep_multiplier.
GetInt())).RetryOnException(
attempt_upload,
sleep_ms=properties.VALUES.storage.
base_retry_delay.GetInt() * 1000,
should_retry_if=_handle_resumable_upload_error)
tracker_data = tracker_file_util.read_resumable_upload_tracker_file(
tracker_file_path)
if tracker_data is not None:
if self._component_number is not None:
tracker_file_util.write_resumable_upload_tracker_file(
tracker_file_path,
complete=True,
encryption_key_sha256=tracker_data.
encryption_key_sha256,
serialization_data=tracker_data.serialization_data)
else:
tracker_file_util.delete_tracker_file(
tracker_file_path)
else:
destination_resource = api.upload_object(
source_stream,
self._destination_resource,
request_config,
source_resource=source_resource_for_metadata,
upload_strategy=upload_strategy)
upload_util.validate_uploaded_object(digesters,
destination_resource,
task_status_queue)
return self._get_output(destination_resource)
def helper(*_): self.assertIsInstance(api_factory.get_api(None), mock.Mock)
def execute(self, callback=None):
provider = self._object_resource.storage_url.scheme
api_factory.get_api(provider).delete_object(self._object_resource)
