class Consumer: """ Consumer for simple message queue. """ def __init__(self, conn, concurrency=1, qname=None, loop=None): self._qname = qname or settings.DEFAULT_Q self._queue = RedisQueue(conn, 'test') self.concurrency = concurrency self._loop = loop or asyncio.get_event_loop() self._is_running = True def wait_for_data(self): """Fetch data from queue ant pass it to next user.""" task = asyncio.Task(self._queue.get(), loop=self._loop) return task def _create_generator(self): """ Creates infinite generator. Method pulls data from queue and yields it for further processing. :return: ``iterable``generator to generate jobs """ while self._is_running: task = self.wait_for_data() yield task @asyncio.coroutine def return_msg_back(self, msg): yield from self._queue.put(msg) @asyncio.coroutine def cooperate(self, iter): for future in iter: msg_raw = yield from future logger.debug("Goe message from queue: {}".format(msg_raw)) try: yield from self._dispatch(msg_raw) except (TMQNoNamespaceHandler, TMQNoActionMethod): log_msg = 'No handlers for msg: {}'.format(msg_raw) logger.error(log_msg) except Exception as e: # TODO: should be better way logger.error('Unhandled error in occurred: {},' 'wait {} sec until next job'.format(e, 5)) yield self.return_msg_back(msg_raw) yield from asyncio.sleep(5, loop=self._loop) def _dispatch(self, msg_raw): job = Job(msg_raw) handle_class = MetaRegister.REGISTRY.get(job.namespace, None) if not handle_class: logger.warning("Namespace does not recognized, there is no" "handler class for this namespace ?") raise TMQNoNamespaceHandler() handler = handle_class(job) yield from handler._handle() def work(self, generator=None): """Start listening and processing tasks""" job_gen = generator or self._create_generator() futures = [] for i in range(self.concurrency): f = asyncio.async(self.cooperate(job_gen), loop=self._loop) futures.append(f) return futures