def _configure(self): self.coord = retry_on_exception(utils.get_coordinator_and_start, self.conf.storage.coordination_url) self.store = retry_on_exception(storage.get_driver, self.conf, self.coord) self.incoming = retry_on_exception(incoming.get_driver, self.conf) self.index = retry_on_exception(indexer.get_driver, self.conf) # create fallback in case paritioning fails or assigned no tasks self.fallback_tasks = list(six.moves.range(self.incoming.NUM_SACKS)) try: self.partitioner = self.coord.join_partitioned_group( self.GROUP_ID, partitions=200) LOG.info('Joined coordination group: %s', self.GROUP_ID) except NotImplementedError: LOG.warning('Coordinator does not support partitioning. Worker ' 'will battle against other workers for jobs.') except tooz.ToozError as e: LOG.error( 'Unexpected error configuring coordinator for ' 'partitioning. Retrying: %s', e) raise tenacity.TryAgain(e) filler = threading.Thread(target=self._fill_sacks_to_process) filler.daemon = True filler.start()
async def msgqueue(scope): with suppress(asyncio.CancelledError): async with req("get", "v1/msgqueues/%s" % scope, timeout=None) as res: logger.info("Getting messages from scope %s", scope) async for message in reader(res): if message["type"] == "heartbeat": continue categ, cmd = message["type"].split(":") callback = event_handlers[scope].get(categ, {}).get(cmd) if callback is None: logger.warning( "Cannot handle event type \"%s\" for queue %s", message["type"], scope) continue logger.debug("Event %s recieved !", message["type"]) view.footer.set_text("Event %s recieved !" % message["type"]) del message["type"] if asyncio.iscoroutinefunction(callback): asyncio.ensure_future(callback(message)) else: callback(message) logger.info("End of stream for scope %s", scope) raise tenacity.TryAgain()
def _fill_sacks_to_process(self): try: for sack in self.incoming.iter_on_sacks_to_process(): if sack in self._get_sacks_to_process(): LOG.debug( "Got notification for sack %s, waking up processing", sack) self.sacks_with_measures_to_process.add(sack) self.wakeup() except exceptions.NotImplementedError: LOG.info("Incoming driver does not support notification") except Exception as e: LOG.error( "Error while listening for new measures notification, " "retrying", exc_info=True) raise tenacity.TryAgain(e)
def _configure(self): self.store = storage.get_driver(self.conf, self.coord) self.incoming = incoming.get_driver(self.conf) self.index = indexer.get_driver(self.conf) self.index.connect() # create fallback in case paritioning fails or assigned no tasks self.fallback_tasks = list(six.moves.range(self.incoming.NUM_SACKS)) try: self.partitioner = self.coord.join_partitioned_group( self.GROUP_ID, partitions=200) LOG.info('Joined coordination group: %s', self.GROUP_ID) except NotImplementedError: LOG.warning('Coordinator does not support partitioning. Worker ' 'will battle against other workers for jobs.') except tooz.ToozError as e: LOG.error( 'Unexpected error configuring coordinator for ' 'partitioning. Retrying: %s', e) raise tenacity.TryAgain(e)
def _configure(self): super(MetricProcessor, self)._configure() # create fallback in case paritioning fails or assigned no tasks self.fallback_tasks = list(six.moves.range(self.incoming.NUM_SACKS)) try: self.partitioner = self.coord.join_partitioned_group( self.GROUP_ID, partitions=200) LOG.info('Joined coordination group: %s', self.GROUP_ID.decode()) except tooz.NotImplemented: LOG.warning('Coordinator does not support partitioning. Worker ' 'will battle against other workers for jobs.') except tooz.ToozError as e: LOG.error( 'Unexpected error configuring coordinator for ' 'partitioning. Retrying: %s', e) raise tenacity.TryAgain(e) filler = threading.Thread(target=self._fill_sacks_to_process) filler.daemon = True filler.start()
def _enable_coordination(coord): try: coord.start(start_heart=True) except Exception as e: LOG.error("Unable to start coordinator: %s", e) raise tenacity.TryAgain(e)
def _check_rmq_running(): logging.info("Waiting for RabbitMQ to start") if not self.rabbit_running: raise tenacity.TryAgain() else: logging.info("RabbitMQ started")