def _get_output(self, destination_resource):
messages = []
if self._component_number is not None:
messages.append(
task.Message(topic=task.Topic.UPLOADED_COMPONENT,
payload=UploadedComponent(
component_number=self._component_number,
object_resource=destination_resource)))
else:
messages.append(
task.Message(topic=task.Topic.CREATED_RESOURCE,
payload=destination_resource))
return task.Output(additional_task_iterators=None, messages=messages)
def _get_output(self, digesters, api_download_result):
"""Generates task.Output from download execution results.
Args:
digesters (dict): Contains hash objects for download checksums.
api_download_result (cloud_api.DownloadApiClientReturnValue|None): Generic
information from API client about the download results.
Returns:
task.Output: Data the parent download or finalize download class would
like to have.
"""
messages = []
if hash_util.HashAlgorithm.MD5 in digesters:
md5_digest = hash_util.get_base64_hash_digest_string(
digesters[hash_util.HashAlgorithm.MD5])
messages.append(
task.Message(topic=task.Topic.MD5, payload=md5_digest))
if hash_util.HashAlgorithm.CRC32C in digesters:
crc32c_checksum = crc32c.get_checksum(
digesters[hash_util.HashAlgorithm.CRC32C])
messages.append(
task.Message(topic=task.Topic.CRC32C,
payload={
'component_number': self._component_number,
'crc32c_checksum': crc32c_checksum,
'length': self._length,
}))
if (api_download_result and self._user_request_args
and self._user_request_args.system_posix_data):
messages.append(
task.Message(topic=task.Topic.API_DOWNLOAD_RESULT,
payload=api_download_result))
return task.Output(additional_task_iterators=None, messages=messages)
def execute(self, task_status_queue=None):
del task_status_queue # Unused.
request_config = request_config_factory.get_request_config(
self._destination_resource.storage_url,
user_request_args=self._user_request_args)
provider = self._destination_resource.storage_url.scheme
created_resource = api_factory.get_api(provider).compose_objects(
self._source_resources,
self._destination_resource,
request_config,
original_source_resource=self._original_source_resource)
return task.Output(messages=[
task.Message(topic=task.Topic.CREATED_RESOURCE,
payload=created_resource),
],
additional_task_iterators=[])
def _thread_worker(task_queue, task_output_queue, task_status_queue,
idle_thread_count):
"""A consumer thread run in a child process.
Args:
task_queue (multiprocessing.Queue): Holds task_graph.TaskWrapper instances.
task_output_queue (multiprocessing.Queue): Sends information about completed
tasks back to the main process.
task_status_queue (multiprocessing.Queue|None): Used by task to report it
progress to a central location.
idle_thread_count (multiprocessing.Semaphore): Keeps track of how many
threads are busy. Useful for spawning new workers if all threads are busy.
"""
while True:
with _task_queue_lock():
task_wrapper = task_queue.get()
if task_wrapper == _SHUTDOWN:
break
idle_thread_count.acquire()
task_execution_error = None
try:
task_output = task_wrapper.task.execute(
task_status_queue=task_status_queue)
# pylint: disable=broad-except
# If any exception is raised, it will prevent the executor from exiting.
except Exception as exception:
task_execution_error = exception
log.error(exception)
log.debug(exception, exc_info=sys.exc_info())
if task_wrapper.task.report_error:
task_output = task.Output(additional_task_iterators=None,
messages=[
task.Message(
topic=task.Topic.FATAL_ERROR,
payload={})
])
else:
task_output = None
# pylint: enable=broad-except
finally:
task_wrapper.task.exit_handler(task_execution_error,
task_status_queue)
task_output_queue.put((task_wrapper, task_output))
idle_thread_count.release()
def execute(self, task_status_queue=None):
"""Performs download."""
digesters = _get_digesters(self._component_number,
self._source_resource)
progress_callback = progress_callbacks.FilesAndBytesProgressCallback(
status_queue=task_status_queue,
offset=self._offset,
length=self._length,
source_url=self._source_resource.storage_url,
destination_url=self._destination_resource.storage_url,
component_number=self._component_number,
total_components=self._total_components,
operation_name=task_status.OperationName.DOWNLOADING,
process_id=os.getpid(),
thread_id=threading.get_ident(),
)
request_config = request_config_factory.get_request_config(
self._source_resource.storage_url,
decryption_key_hash=self._source_resource.decryption_key_hash,
user_request_args=self._user_request_args,
)
if self._source_resource.size and self._component_number is not None:
try:
api_download_result = self._perform_component_download(
request_config, progress_callback, digesters)
# pylint:disable=broad-except
except Exception as e:
# pylint:enable=broad-except
return task.Output(
additional_task_iterators=None,
messages=[task.Message(topic=task.Topic.ERROR, payload=e)])
elif self._strategy is cloud_api.DownloadStrategy.RESUMABLE:
api_download_result = self._perform_resumable_download(
request_config, progress_callback, digesters)
else:
api_download_result = self._perform_one_shot_download(
request_config, progress_callback, digesters)
return self._get_output(digesters, api_download_result)