def test_print_quiet(self):
print_task = PrintTask(message="Success", error=False)
# Ensure a succesful task is not printed to stdout when
# ``quiet`` is True.
thread = PrintThread(result_queue=self.result_queue,
quiet=True,
only_show_errors=False)
self.assert_expected_output(print_task, '', thread, 'sys.stdout')
def test_print_warning_only_show_errors(self):
print_task = PrintTask(message="Bad File.", warning=True)
# Ensure a warned task is printed to stderr when
# ``only_show_errors`` is True.
thread = PrintThread(result_queue=self.result_queue,
quiet=False, only_show_errors=True)
self.assert_expected_output(print_task, 'Bad File.', thread,
'sys.stderr')
def test_print_error(self):
print_task = PrintTask(message="Fail File.", error=True)
# Ensure a failed task is printed to stderr when
# ``quiet`` and ``only_show_errors`` is False.
thread = PrintThread(result_queue=self.result_queue,
quiet=False, only_show_errors=False)
self.assert_expected_output(print_task, 'Fail File.', thread,
'sys.stderr')
def test_print_decoding_error_message(self):
# This ensures that if the message being passed to the print string
# is from a decoding error. It can be outputted properly.
decoding_error = FileDecodingError('temp', b'\xe2\x9c\x93')
print_task = PrintTask(message=decoding_error.error_message,
warning=True)
thread = PrintThread(result_queue=self.result_queue,
quiet=False,
only_show_errors=False)
self.assert_expected_output(print_task, decoding_error.error_message,
thread, 'sys.stderr')
def _queue_print_message(self, filename, failed, dryrun,
error_message=None):
try:
if filename.operation_name != 'list_objects':
message = print_operation(filename, failed,
self.parameters['dryrun'])
if error_message is not None:
message += ' ' + error_message
result = {'message': message, 'error': failed}
self.result_queue.put(PrintTask(**result))
except Exception as e:
LOGGER.debug('%s' % str(e))
def _download_part(self):
total_file_size = self._filename.size
start_range = self._part_number * self._chunk_size
if self._part_number == int(total_file_size / self._chunk_size) - 1:
end_range = ''
else:
end_range = start_range + self._chunk_size - 1
range_param = 'bytes=%s-%s' % (start_range, end_range)
LOGGER.debug("Downloading bytes range of %s for file %s", range_param,
self._filename.dest)
bucket, key = find_bucket_key(self._filename.src)
params = {
'endpoint': self._filename.endpoint,
'bucket': bucket,
'key': key,
'range': range_param
}
for i in range(self.TOTAL_ATTEMPTS):
try:
LOGGER.debug("Making GetObject requests with byte range: %s",
range_param)
response_data, http = operate(self._service, 'GetObject',
params)
LOGGER.debug("Response received from GetObject")
body = response_data['Body']
self._queue_writes(body)
self._context.announce_completed_part(self._part_number)
message = print_operation(self._filename, 0)
total_parts = int(self._filename.size / self._chunk_size)
result = {
'message': message,
'error': False,
'total_parts': total_parts
}
self._result_queue.put(PrintTask(**result))
LOGGER.debug("Task complete: %s", self)
return
except (socket.timeout, socket.error) as e:
LOGGER.debug(
"Socket timeout caught, retrying request, "
"(attempt %s / %s)",
i,
self.TOTAL_ATTEMPTS,
exc_info=True)
continue
except IncompleteReadError as e:
LOGGER.debug("Incomplete read detected: %s, (attempt %s / %s)",
e, i, self.TOTAL_ATTEMPTS)
continue
raise RetriesExeededError("Maximum number of attempts exceeded: %s" %
self.TOTAL_ATTEMPTS)
def call(self, files):
"""
This function pulls a ``FileInfo`` or ``TaskInfo`` object from
a list ``files``. Each object is then deemed if it will be a
multipart operation and add the necessary attributes if so. Each
object is then wrapped with a ``BasicTask`` object which is
essentially a thread of execution for a thread to follow. These
tasks are then submitted to the main executor.
"""
try:
self.executor.start()
total_files, total_parts = self._enqueue_tasks(files)
self.executor.print_thread.set_total_files(total_files)
self.executor.print_thread.set_total_parts(total_parts)
self.executor.initiate_shutdown()
self.executor.wait_until_shutdown()
self._shutdown()
except Exception as e:
LOGGER.debug('Exception caught during task execution: %s',
str(e),
exc_info=True)
self.result_queue.put(PrintTask(message=str(e), error=True))
self.executor.initiate_shutdown(
priority=self.executor.IMMEDIATE_PRIORITY)
self._shutdown()
self.executor.wait_until_shutdown()
except KeyboardInterrupt:
self.result_queue.put(
PrintTask(message=("Cleaning up. "
"Please wait..."),
error=True))
self.executor.initiate_shutdown(
priority=self.executor.IMMEDIATE_PRIORITY)
self._shutdown()
self.executor.wait_until_shutdown()
return CommandResult(self.executor.num_tasks_failed,
self.executor.num_tasks_warned)
def __call__(self):
try:
self._download_part()
except Exception as e:
LOGGER.debug(
'Exception caught downloading byte range: %s',
e, exc_info=True)
message = print_operation(self._filename, failed=True,
dryrun=False)
message += '\n' + str(e)
result = {'message': message, 'error': True}
self._result_queue.put(PrintTask(**result))
self._context.cancel()
raise e
def __call__(self):
# When the file is downloading, we have a few things we need to do:
# 1) Fix up the last modified time to match s3.
# 2) Tell the result_queue we're done.
# 3) Queue an IO request to the IO thread letting it know we're
# done with the file.
self._context.wait_for_completion()
last_update_tuple = self._filename.last_update.timetuple()
mod_timestamp = time.mktime(last_update_tuple)
desired_mtime = int(mod_timestamp)
message = print_operation(self._filename, False,
self._parameters['dryrun'])
print_task = {'message': message, 'error': False}
self._result_queue.put(PrintTask(**print_task))
self._io_queue.put(IOCloseRequest(self._filename.dest, desired_mtime))
def __call__(self):
LOGGER.debug("Creating multipart upload for file: %s",
self.filename.src)
try:
upload_id = self.filename.create_multipart_upload()
LOGGER.debug("Announcing upload id: %s", upload_id)
self._upload_context.announce_upload_id(upload_id)
except Exception as e:
LOGGER.debug("Error trying to create multipart upload: %s",
e, exc_info=True)
self._upload_context.cancel_upload()
message = print_operation(self.filename, True,
self.parameters['dryrun'])
message += '\n' + str(e)
result = {'message': message, 'error': True}
self.result_queue.put(PrintTask(**result))
raise e
