async def _may_poll_task(self, worker): # Called in the ``worker`` event loop. # # It pools a new task if possible, and add it to the queue of # tasks consumed by the ``worker`` CPU-bound thread.''' task = None next_time = None lag = 0 if worker.is_running(): loop = worker._loop if self.num_concurrent_tasks < self.max_concurrent_tasks: max_tasks = self.cfg.max_requests if max_tasks and self._processed >= max_tasks: self.backend.close( 'Processed %s tasks. Stop polling tasks.' % self._processed) if not self.closing(): try: t0 = loop.time() task = await self.broker.get_message(*self.queues()) lag = loop.time() - t0 except ConnectionError: if self.broker.connection_error: next_time = backoff(self._next_time) else: next_time = RECONNECT_LAG self.broker.connection_error = True if worker.is_running(): self.logger.critical( '%s cannot pool messages - ' 'connection error - try again in %s seconds', self.broker, next_time) except CANCELLED_ERRORS: self.logger.debug('stopped polling messages') raise except Exception: if worker.is_running(): self.logger.exception('server exception') else: self.broker.connection_ok() if task: # Got a new task self._processed += 1 self._concurrent_tasks[task.id] = TaskExecutor(task) ensure_future(self._execute_task(task, worker)) else: self.logger.debug('%s concurrent messages. Cannot poll.', self.max_concurrent_tasks) if next_time is None: next_time = poll_time( self.cfg.task_pool_timeout, self.cfg.task_pool_timeout_max, self.num_concurrent_tasks / self.max_concurrent_tasks, lag) self._next_time = next_time self._poll_tasks(worker, next_time)
async def worker_tick(self, worker, next=None): pnext, next = next, HEARTBEAT try: info = dict(self.info()) info['consumer'] = worker.aid info['node'] = self.node_name info['pubsub'] = str(self.channels) info['cores'] = cpu_count() info['message-broker'] = str(self.broker.store) info['time'] = time.time() if self.cfg.debug: self.logger.debug('publishing worker %s info', worker) await self.publish('status', ConsumerMessage(info)) except ConnectionError: next = next if not pnext else backoff(pnext) self.logger.critical( 'Cannot publish consumer status: connection error.' ' Try in %s seconds', next ) finally: worker._loop.call_later(next, self.__tick, worker, next)
async def _may_poll_task(self, worker): # Called in the ``worker`` event loop. # # It pools a new task if possible, and add it to the queue of # tasks consumed by the ``worker`` CPU-bound thread.''' task = None next_time = None lag = 0 if worker.is_running(): loop = worker._loop if self.num_concurrent_tasks < self.max_concurrent_tasks: max_tasks = self.cfg.max_requests if max_tasks and self._processed >= max_tasks: self.backend.close( 'Processed %s tasks. Stop polling tasks.' % self._processed ) if not self.closing(): try: t0 = loop.time() task = await self.broker.get_message(*self.queues()) lag = loop.time() - t0 except ConnectionError: if self.broker.connection_error: next_time = backoff(self._next_time) else: next_time = RECONNECT_LAG self.broker.connection_error = True if worker.is_running(): self.logger.critical( '%s cannot pool messages - ' 'connection error - try again in %s seconds', self.broker, next_time ) except CANCELLED_ERRORS: self.logger.debug('stopped polling messages') raise except Exception: if worker.is_running(): self.logger.exception('server exception') else: self.broker.connection_ok() if task: # Got a new task self._processed += 1 self._concurrent_tasks[task.id] = TaskExecutor(task) ensure_future(self._execute_task(task, worker)) else: self.logger.debug('%s concurrent messages. Cannot poll.', self.max_concurrent_tasks) if next_time is None: next_time = poll_time( self.cfg.task_pool_timeout, self.cfg.task_pool_timeout_max, self.num_concurrent_tasks/self.max_concurrent_tasks, lag ) self._next_time = next_time self._poll_tasks(worker, next_time)