def apply_async(self, target, args=[], kwargs={}, **options): """Equivalent of the :func:`apply` built-in function. Callbacks should optimally return as soon as possible since otherwise the thread which handles the result will get blocked. """ if self._does_debug: logger.debug('TaskPool: Apply %s (args:%s kwargs:%s)', target, truncate(safe_repr(args), 1024), truncate(safe_repr(kwargs), 1024)) return self.on_apply(target, args, kwargs, waitforslot=self.putlocks, callbacks_propagate=self.callbacks_propagate, **options)
def dump_body(m, body): # v2 protocol does not deserialize body body = m.body if body is None else body if isinstance(body, buffer_t): body = bytes_t(body) return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def join(self, r, propagate=False, max_retries=10, **kwargs): if self.no_join: return received = [] def on_result(task_id, value): received.append(task_id) for i in range(max_retries) if max_retries else count(0): received[:] = [] try: return r.get(callback=on_result, propagate=propagate, **kwargs) except (socket.timeout, TimeoutError) as exc: waiting_for = self.missing_results(r) self.speaker.beep() marker( 'Still waiting for {0}/{1}: [{2}]: {3!r}'.format( len(r) - len(received), len(r), truncate(', '.join(waiting_for)), exc), '!', ) self.fbi.diag(waiting_for) except self.connerrors as exc: self.speaker.beep() marker('join: connection lost: {0!r}'.format(exc), '!') raise StopSuite('Test failed: Missing task results')
def repr_result(self, result, maxlen=46): # 46 is the length needed to fit # 'the quick brown fox jumps over the lazy dog' :) return truncate(safe_repr(result), maxlen)
def dump_body(m, body): if isinstance(body, buffer_t): body = bytes_t(buffer) return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def trace_task(uuid, args, kwargs, request=None): # R - is the possibly prepared return value. # I - is the Info object. # T - runtime # Rstr - textual representation of return value # retval - is the always unmodified return value. # state - is the resulting task state. # This function is very long because we have unrolled all the calls # for performance reasons, and because the function is so long # we want the main variables (I, and R) to stand out visually from the # the rest of the variables, so breaking PEP8 is worth it ;) R = I = T = Rstr = retval = state = None task_request = None time_start = monotonic() try: try: kwargs.items except AttributeError: raise InvalidTaskError( 'Task keyword arguments is not a mapping') push_task(task) task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) push_request(task_request) try: # -*- PRE -*- if prerun_receivers: send_prerun(sender=task, task_id=uuid, task=task, args=args, kwargs=kwargs) loader_task_init(uuid, task) if track_started: store_result( uuid, {'pid': pid, 'hostname': hostname}, STARTED, request=task_request, ) # -*- TRACE -*- try: R = retval = fun(*args, **kwargs) state = SUCCESS except Reject as exc: I, R = Info(REJECTED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval I.handle_reject(task, task_request) except Ignore as exc: I, R = Info(IGNORED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval I.handle_ignore(task, task_request) except Retry as exc: I, R, state, retval = on_error( task_request, exc, uuid, RETRY, call_errbacks=False, ) except Exception as exc: I, R, state, retval = on_error(task_request, exc, uuid) if task_request.chord: on_chord_part_return(task, state, exc) except BaseException as exc: raise else: try: # callback tasks must be applied before the result is # stored, so that result.children is populated. # groups are called inline and will store trail # separately, so need to call them separately # so that the trail's not added multiple times :( # (Issue #1936) callbacks = task.request.callbacks if callbacks: if len(task.request.callbacks) > 1: sigs, groups = [], [] for sig in callbacks: sig = signature(sig, app=app) if isinstance(sig, group): groups.append(sig) else: sigs.append(sig) for group_ in groups: group.apply_async((retval, )) if sigs: group(sigs).apply_async((retval, )) else: signature(callbacks[0], app=app).delay(retval) if publish_result: store_result( uuid, retval, SUCCESS, request=task_request, ) except EncodeError as exc: I, R, state, retval = on_error(task_request, exc, uuid) else: if task_request.chord: on_chord_part_return(task, state, retval) if task_on_success: task_on_success(retval, uuid, args, kwargs) if success_receivers: send_success(sender=task, result=retval) if _does_info: T = monotonic() - time_start Rstr = truncate(safe_repr(R), 256) info(LOG_SUCCESS, { 'id': uuid, 'name': name, 'return_value': Rstr, 'runtime': T, }) # -* POST *- if state not in IGNORE_STATES: if task_after_return: task_after_return( state, retval, uuid, args, kwargs, None, ) finally: try: if postrun_receivers: send_postrun(sender=task, task_id=uuid, task=task, args=args, kwargs=kwargs, retval=retval, state=state) finally: pop_task() pop_request() if not eager: try: backend_cleanup() loader_cleanup() except (KeyboardInterrupt, SystemExit, MemoryError): raise except Exception as exc: logger.error('Process cleanup failed: %r', exc, exc_info=True) except MemoryError: raise except Exception as exc: if eager: raise R = report_internal_error(task, exc) if task_request is not None: I, _, _, _ = on_error(task_request, exc, uuid) return trace_ok_t(R, I, T, Rstr)
def test_truncate_text(s, maxsize, expected): assert truncate(s, maxsize) == expected
def test_truncate_text(self): self.assertEqual(truncate('ABCDEFGHI', 3), 'ABC...') self.assertEqual(truncate('ABCDEFGHI', 10), 'ABCDEFGHI')
def repr_result(self, result, maxlen=RESULT_MAXLEN): # 46 is the length needed to fit # 'the quick brown fox jumps over the lazy dog' :) if not isinstance(result, string_t): result = safe_repr(result) return truncate(result) if len(result) > maxlen else result
def test_truncate_text(self): self.assertEqual(text.truncate('ABCDEFGHI', 3), 'ABC...') self.assertEqual(text.truncate('ABCDEFGHI', 10), 'ABCDEFGHI')
def prepare_result(self, result, maxlen=RESULT_MAXLEN, truncate=truncate): if not isinstance(result, ExceptionInfo): return truncate(_repr_result(result), maxlen) return result
def test_truncate_text(self): self.assertEqual(text.truncate("ABCDEFGHI", 3), "ABC...") self.assertEqual(text.truncate("ABCDEFGHI", 10), "ABCDEFGHI")
def prepare_result(self, result, RESULT_MAXLEN=RESULT_MAXLEN): if not isinstance(result, ExceptionInfo): return truncate(repr(result), RESULT_MAXLEN) return result
def __repr__(self): task, it = self._unpack_args(self.kwargs) return '[{0}(*x) for x in {1}]'.format( task.task, truncate(repr(it), 100))
def __str__(self): return truncate(self.body, maxlen=30)
def __repr__(self): task, it = self._unpack_args(self.kwargs) return '[%s(*x) for x in %s]' % (task.task, truncate(repr(it), 100))
def dump_body(m, body): return "{0} ({1}b)".format(text.truncate(safe_repr(body), 1024), len(m.body))
def dump_body(m, body): return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def prepare_result(self, result): if not isinstance(result, ExceptionInfo): return truncate(repr(result), 46)
def dump_body(m, body): """Format message body for debugging purposes.""" # v2 protocol does not deserialize body body = m.body if body is None else body return '{} ({}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def repr_result(self, result, maxlen=46): # 46 is the length needed to fit # 'the quick brown fox jumps over the lazy dog' :) if not isinstance(result, string_t): result = safe_repr(result) return truncate(result) if len(result) > maxlen else result
def dump_body(m, body): if isinstance(body, buffer_t): body = bytes_t(body) return '{0} ({1}b)'.format(truncate(safe_repr(body), 1024), len(m.body))
def __repr__(self): task, it = self._unpack_args(self.kwargs) return '[{0}(*x) for x in {1}]'.format(task.task, truncate(repr(it), 100))
def trace_task(uuid, args, kwargs, request=None): # R - is the possibly prepared return value. # I - is the Info object. # T - runtime # Rstr - textual representation of return value # retval - is the always unmodified return value. # state - is the resulting task state. # This function is very long because we have unrolled all the calls # for performance reasons, and because the function is so long # we want the main variables (I, and R) to stand out visually from the # the rest of the variables, so breaking PEP8 is worth it ;) R = I = T = Rstr = retval = state = None task_request = None time_start = monotonic() try: try: kwargs.items except AttributeError: raise InvalidTaskError( 'Task keyword arguments is not a mapping') push_task(task) task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs) push_request(task_request) try: # -*- PRE -*- if prerun_receivers: send_prerun(sender=task, task_id=uuid, task=task, args=args, kwargs=kwargs) loader_task_init(uuid, task) if track_started: store_result( uuid, {'pid': pid, 'hostname': hostname}, STARTED, request=task_request, ) # -*- TRACE -*- try: R = retval = fun(*args, **kwargs) state = SUCCESS except Reject as exc: I, R = Info(REJECTED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval I.handle_reject(task, task_request) except Ignore as exc: I, R = Info(IGNORED, exc), ExceptionInfo(internal=True) state, retval = I.state, I.retval I.handle_ignore(task, task_request) except Retry as exc: I, R, state, retval = on_error( task_request, exc, uuid, RETRY, call_errbacks=False, ) except Exception as exc: I, R, state, retval = on_error(task_request, exc, uuid) except BaseException as exc: raise else: try: # callback tasks must be applied before the result is # stored, so that result.children is populated. # groups are called inline and will store trail # separately, so need to call them separately # so that the trail's not added multiple times :( # (Issue #1936) callbacks = task.request.callbacks if callbacks: if len(task.request.callbacks) > 1: sigs, groups = [], [] for sig in callbacks: sig = signature(sig, app=app) if isinstance(sig, group): groups.append(sig) else: sigs.append(sig) for group_ in groups: group.apply_async((retval,)) if sigs: group(sigs).apply_async((retval,)) else: signature(callbacks[0], app=app).delay(retval) # execute first task in chain chain = task.request.chain if chain: signature(chain.pop(), app=app).apply_async( (retval,), chain=chain) mark_as_done( uuid, retval, task_request, publish_result, ) except EncodeError as exc: I, R, state, retval = on_error(task_request, exc, uuid) else: if task_on_success: task_on_success(retval, uuid, args, kwargs) if success_receivers: send_success(sender=task, result=retval) if _does_info: T = monotonic() - time_start Rstr = truncate(safe_repr(R), 256) info(LOG_SUCCESS, { 'id': uuid, 'name': name, 'return_value': Rstr, 'runtime': T, }) # -* POST *- if state not in IGNORE_STATES: if task_after_return: task_after_return( state, retval, uuid, args, kwargs, None, ) finally: try: if postrun_receivers: send_postrun(sender=task, task_id=uuid, task=task, args=args, kwargs=kwargs, retval=retval, state=state) finally: pop_task() pop_request() if not eager: try: backend_cleanup() loader_cleanup() except (KeyboardInterrupt, SystemExit, MemoryError): raise except Exception as exc: logger.error('Process cleanup failed: %r', exc, exc_info=True) except MemoryError: raise except Exception as exc: if eager: raise R = report_internal_error(task, exc) if task_request is not None: I, _, _, _ = on_error(task_request, exc, uuid) return trace_ok_t(R, I, T, Rstr)
def __str__(self): return truncate(self.title, maxlen=30)