def _run_worker(self, worker_ctx, handle_result):
_log.debug('setting up %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
if not worker_ctx.parent_call_stack:
_log.debug('starting call chain',
extra=worker_ctx.extra_for_logging)
_log.debug('call stack for %s: %s',
worker_ctx, '->'.join(worker_ctx.call_id_stack),
extra=worker_ctx.extra_for_logging)
with log_time(_log.debug, 'ran worker %s in %0.3fsec', worker_ctx):
self.dependencies.injections.all.inject(worker_ctx)
self.dependencies.all.worker_setup(worker_ctx)
result = exc = None
try:
_log.debug('calling handler for %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
method = getattr(worker_ctx.service, worker_ctx.method_name)
with log_time(_log.debug, 'ran handler for %s in %0.3fsec',
worker_ctx):
result = method(*worker_ctx.args, **worker_ctx.kwargs)
except Exception as e:
log_worker_exception(worker_ctx, e)
exc = e
with log_time(_log.debug, 'tore down worker %s in %0.3fsec',
worker_ctx):
_log.debug('signalling result for %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
self.dependencies.injections.all.worker_result(
worker_ctx, result, exc)
_log.debug('tearing down %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
self.dependencies.all.worker_teardown(worker_ctx)
self.dependencies.injections.all.release(worker_ctx)
if handle_result is not None:
_log.debug('handling result for %s', worker_ctx,
extra=worker_ctx.extra_for_logging)
with log_time(_log.debug, 'handled result for %s in %0.3fsec',
worker_ctx):
handle_result(worker_ctx, result, exc)
def start(self):
""" Start a container by starting all the dependency providers.
"""
_log.debug('starting %s', self)
self.started = True
with log_time(_log.debug, 'started %s in %0.3f sec', self):
self.dependencies.all.prepare()
self.dependencies.all.start()
def reply(connection, msg_id, replydata=None, failure=None, on_return=None):
_log.debug('replying to RPC message `%s`', msg_id)
with log_time(
_log.debug, 'replied to message `%s` in %0.3f sec.', msg_id):
if on_return is not None:
raise NotImplementedError('on_return is not implemented')
msg = {'result': replydata, 'failure': failure, 'ending': False, }
send_direct(connection, msg_id, msg)
msg = {'result': None, 'failure': None, 'ending': True, }
send_direct(connection, msg_id, msg)
def stop(self):
""" Stop the container gracefully.
First all entrypoints are asked to ``stop()``.
This ensures that no new worker threads are started.
It is the providers' responsibility to gracefully shut down when
``stop()`` is called on them and only return when they have stopped.
After all entrypoints have stopped the container waits for any
active workers to complete.
After all active workers have stopped the container stops all
injections.
At this point there should be no more managed threads. In case there
are any managed threads, they are killed by the container.
"""
if self._died.ready():
_log.debug('already stopped %s', self)
return
_log.debug('stopping %s', self)
with log_time(_log.debug, 'stopped %s in %0.3f sec', self):
dependencies = self.dependencies
# entrypoint deps have to be stopped before injection deps
# to ensure that running workers can successfully complete
dependencies.entrypoints.all.stop()
# there might still be some running workers, which we have to
# wait for to complete before we can stop injection dependencies
self._worker_pool.waitall()
# it should be safe now to stop any injection as there is no
# active worker which could be using it
dependencies.injections.all.stop()
# finally, stop nested dependencies
dependencies.nested.all.stop()
# just in case there was a provider not taking care of its workers,
# or a dependency not taking care of its protected threads
self._kill_active_threads()
self._kill_protected_threads()
self.started = False
self._died.send(None)
def process_rpc_message(connection, delegate, body):
msgid, ctx, method, args = context.parse_message(body)
_log.debug('processing RPC message `%s`: using %s(...)', msgid, method)
with log_time(
_log.debug, 'processed RPC message `%s` in %0.3f sec.', msgid):
try:
ret = _delegate_apply(delegate, ctx, method, args)
except Exception:
exc_typ, exc_val, exc_tb = sys.exc_info()
if msgid:
tbfmt = traceback.format_exception(exc_typ, exc_val, exc_tb)
tbfmt = ''.join(tbfmt)
ret = (exc_typ.__name__, str(exc_val), tbfmt)
reply(connection, msgid, failure=ret)
else:
if msgid:
reply(connection, msgid, replydata=ret)
def handle_consume_message(self, consumer_method_config, body, message):
with log_time(_log.debug, 'processed consume message in %0.3fsec'):
consumer_method, consumer_config = consumer_method_config
try:
consumer_method(body)
except Exception as e:
if consumer_config.requeue_on_error:
_log.exception(
'failed to consume message, requeueing message: '
'%s(): %s', consumer_method, e)
self._pending_requeue_messages.append(message)
else:
_log.exception(
'failed to consume message, ignoring message: '
'%s(): %s', consumer_method, e)
self._pending_ack_messages.append(message)
else:
self._pending_ack_messages.append(message)
