async def _may_poll_task(self, worker):
# Called in the ``worker`` event loop.
#
# It pools a new task if possible, and add it to the queue of
# tasks consumed by the ``worker`` CPU-bound thread.'''
task = None
next_time = None
lag = 0
if worker.is_running():
loop = worker._loop
if self.num_concurrent_tasks < self.max_concurrent_tasks:
max_tasks = self.cfg.max_requests
if max_tasks and self._processed >= max_tasks:
self.backend.close(
'Processed %s tasks. Stop polling tasks.' %
self._processed)
if not self.closing():
try:
t0 = loop.time()
task = await self.broker.get_message(*self.queues())
lag = loop.time() - t0
except ConnectionError:
if self.broker.connection_error:
next_time = backoff(self._next_time)
else:
next_time = RECONNECT_LAG
self.broker.connection_error = True
if worker.is_running():
self.logger.critical(
'%s cannot pool messages - '
'connection error - try again in %s seconds',
self.broker, next_time)
except CANCELLED_ERRORS:
self.logger.debug('stopped polling messages')
raise
except Exception:
if worker.is_running():
self.logger.exception('server exception')
else:
self.broker.connection_ok()
if task: # Got a new task
self._processed += 1
self._concurrent_tasks[task.id] = TaskExecutor(task)
ensure_future(self._execute_task(task, worker))
else:
self.logger.debug('%s concurrent messages. Cannot poll.',
self.max_concurrent_tasks)
if next_time is None:
next_time = poll_time(
self.cfg.task_pool_timeout, self.cfg.task_pool_timeout_max,
self.num_concurrent_tasks / self.max_concurrent_tasks, lag)
self._next_time = next_time
self._poll_tasks(worker, next_time)
async def worker_tick(self, worker, next=None):
pnext, next = next, HEARTBEAT
try:
info = dict(self.info())
info['consumer'] = worker.aid
info['node'] = self.node_name
info['pubsub'] = str(self.channels)
info['cores'] = cpu_count()
info['message-broker'] = str(self.broker.store)
info['time'] = time.time()
if self.cfg.debug:
self.logger.debug('publishing worker %s info', worker)
await self.publish('status', ConsumerMessage(info))
except ConnectionError:
next = next if not pnext else backoff(pnext)
self.logger.critical(
'Cannot publish consumer status: connection error.'
' Try in %s seconds', next
)
finally:
worker._loop.call_later(next, self.__tick, worker, next)
async def _may_poll_task(self, worker):
# Called in the ``worker`` event loop.
#
# It pools a new task if possible, and add it to the queue of
# tasks consumed by the ``worker`` CPU-bound thread.'''
task = None
next_time = None
lag = 0
if worker.is_running():
loop = worker._loop
if self.num_concurrent_tasks < self.max_concurrent_tasks:
max_tasks = self.cfg.max_requests
if max_tasks and self._processed >= max_tasks:
self.backend.close(
'Processed %s tasks. Stop polling tasks.'
% self._processed
)
if not self.closing():
try:
t0 = loop.time()
task = await self.broker.get_message(*self.queues())
lag = loop.time() - t0
except ConnectionError:
if self.broker.connection_error:
next_time = backoff(self._next_time)
else:
next_time = RECONNECT_LAG
self.broker.connection_error = True
if worker.is_running():
self.logger.critical(
'%s cannot pool messages - '
'connection error - try again in %s seconds',
self.broker,
next_time
)
except CANCELLED_ERRORS:
self.logger.debug('stopped polling messages')
raise
except Exception:
if worker.is_running():
self.logger.exception('server exception')
else:
self.broker.connection_ok()
if task: # Got a new task
self._processed += 1
self._concurrent_tasks[task.id] = TaskExecutor(task)
ensure_future(self._execute_task(task, worker))
else:
self.logger.debug('%s concurrent messages. Cannot poll.',
self.max_concurrent_tasks)
if next_time is None:
next_time = poll_time(
self.cfg.task_pool_timeout,
self.cfg.task_pool_timeout_max,
self.num_concurrent_tasks/self.max_concurrent_tasks,
lag
)
self._next_time = next_time
self._poll_tasks(worker, next_time)
