def test_print_quiet(self): print_task = PrintTask(message="Success", error=False) # Ensure a succesful task is not printed to stdout when # ``quiet`` is True. thread = PrintThread(result_queue=self.result_queue, quiet=True, only_show_errors=False) self.assert_expected_output(print_task, '', thread, 'sys.stdout')
def test_print_warning_only_show_errors(self): print_task = PrintTask(message="Bad File.", warning=True) # Ensure a warned task is printed to stderr when # ``only_show_errors`` is True. thread = PrintThread(result_queue=self.result_queue, quiet=False, only_show_errors=True) self.assert_expected_output(print_task, 'Bad File.', thread, 'sys.stderr')
def test_print_error(self): print_task = PrintTask(message="Fail File.", error=True) # Ensure a failed task is printed to stderr when # ``quiet`` and ``only_show_errors`` is False. thread = PrintThread(result_queue=self.result_queue, quiet=False, only_show_errors=False) self.assert_expected_output(print_task, 'Fail File.', thread, 'sys.stderr')
def test_print_decoding_error_message(self): # This ensures that if the message being passed to the print string # is from a decoding error. It can be outputted properly. decoding_error = FileDecodingError('temp', b'\xe2\x9c\x93') print_task = PrintTask(message=decoding_error.error_message, warning=True) thread = PrintThread(result_queue=self.result_queue, quiet=False, only_show_errors=False) self.assert_expected_output(print_task, decoding_error.error_message, thread, 'sys.stderr')
def _queue_print_message(self, filename, failed, dryrun, error_message=None): try: if filename.operation_name != 'list_objects': message = print_operation(filename, failed, self.parameters['dryrun']) if error_message is not None: message += ' ' + error_message result = {'message': message, 'error': failed} self.result_queue.put(PrintTask(**result)) except Exception as e: LOGGER.debug('%s' % str(e))
def _download_part(self): total_file_size = self._filename.size start_range = self._part_number * self._chunk_size if self._part_number == int(total_file_size / self._chunk_size) - 1: end_range = '' else: end_range = start_range + self._chunk_size - 1 range_param = 'bytes=%s-%s' % (start_range, end_range) LOGGER.debug("Downloading bytes range of %s for file %s", range_param, self._filename.dest) bucket, key = find_bucket_key(self._filename.src) params = { 'endpoint': self._filename.endpoint, 'bucket': bucket, 'key': key, 'range': range_param } for i in range(self.TOTAL_ATTEMPTS): try: LOGGER.debug("Making GetObject requests with byte range: %s", range_param) response_data, http = operate(self._service, 'GetObject', params) LOGGER.debug("Response received from GetObject") body = response_data['Body'] self._queue_writes(body) self._context.announce_completed_part(self._part_number) message = print_operation(self._filename, 0) total_parts = int(self._filename.size / self._chunk_size) result = { 'message': message, 'error': False, 'total_parts': total_parts } self._result_queue.put(PrintTask(**result)) LOGGER.debug("Task complete: %s", self) return except (socket.timeout, socket.error) as e: LOGGER.debug( "Socket timeout caught, retrying request, " "(attempt %s / %s)", i, self.TOTAL_ATTEMPTS, exc_info=True) continue except IncompleteReadError as e: LOGGER.debug("Incomplete read detected: %s, (attempt %s / %s)", e, i, self.TOTAL_ATTEMPTS) continue raise RetriesExeededError("Maximum number of attempts exceeded: %s" % self.TOTAL_ATTEMPTS)
def call(self, files): """ This function pulls a ``FileInfo`` or ``TaskInfo`` object from a list ``files``. Each object is then deemed if it will be a multipart operation and add the necessary attributes if so. Each object is then wrapped with a ``BasicTask`` object which is essentially a thread of execution for a thread to follow. These tasks are then submitted to the main executor. """ try: self.executor.start() total_files, total_parts = self._enqueue_tasks(files) self.executor.print_thread.set_total_files(total_files) self.executor.print_thread.set_total_parts(total_parts) self.executor.initiate_shutdown() self.executor.wait_until_shutdown() self._shutdown() except Exception as e: LOGGER.debug('Exception caught during task execution: %s', str(e), exc_info=True) self.result_queue.put(PrintTask(message=str(e), error=True)) self.executor.initiate_shutdown( priority=self.executor.IMMEDIATE_PRIORITY) self._shutdown() self.executor.wait_until_shutdown() except KeyboardInterrupt: self.result_queue.put( PrintTask(message=("Cleaning up. " "Please wait..."), error=True)) self.executor.initiate_shutdown( priority=self.executor.IMMEDIATE_PRIORITY) self._shutdown() self.executor.wait_until_shutdown() return CommandResult(self.executor.num_tasks_failed, self.executor.num_tasks_warned)
def __call__(self): try: self._download_part() except Exception as e: LOGGER.debug( 'Exception caught downloading byte range: %s', e, exc_info=True) message = print_operation(self._filename, failed=True, dryrun=False) message += '\n' + str(e) result = {'message': message, 'error': True} self._result_queue.put(PrintTask(**result)) self._context.cancel() raise e
def __call__(self): # When the file is downloading, we have a few things we need to do: # 1) Fix up the last modified time to match s3. # 2) Tell the result_queue we're done. # 3) Queue an IO request to the IO thread letting it know we're # done with the file. self._context.wait_for_completion() last_update_tuple = self._filename.last_update.timetuple() mod_timestamp = time.mktime(last_update_tuple) desired_mtime = int(mod_timestamp) message = print_operation(self._filename, False, self._parameters['dryrun']) print_task = {'message': message, 'error': False} self._result_queue.put(PrintTask(**print_task)) self._io_queue.put(IOCloseRequest(self._filename.dest, desired_mtime))
def __call__(self): LOGGER.debug("Creating multipart upload for file: %s", self.filename.src) try: upload_id = self.filename.create_multipart_upload() LOGGER.debug("Announcing upload id: %s", upload_id) self._upload_context.announce_upload_id(upload_id) except Exception as e: LOGGER.debug("Error trying to create multipart upload: %s", e, exc_info=True) self._upload_context.cancel_upload() message = print_operation(self.filename, True, self.parameters['dryrun']) message += '\n' + str(e) result = {'message': message, 'error': True} self.result_queue.put(PrintTask(**result)) raise e