class Watchdog(object): """ Runs a callable in a gevent greenlet, and restarts the greenlet with the same callable iff any exception is raised from the greenlet. Uses exponential backoff to respawn the greenlet until eventually it gives up. """ def __init__(self, func, backoff=None): if not callable(func): raise ValueError('Func argument is not callable') if not backoff: backoff = TimeSensitiveBackoff() self.func = func self.backoff = backoff def __call__(self, greenlet): try: logger.exception(greenlet.exception) time.sleep(self.backoff.next()) greenlet = self.respawn() return greenlet except StopIteration: _failhard('backoff exceeded', self.func, greenlet) get_hub().parent.throw(SystemExit()) def spawn(self): self.greenlet = Greenlet(self.func) self.greenlet.link_exception(self) self.greenlet.start() return self.greenlet def respawn(self): return self.spawn()
def add_pending_greenlet(self, greenlet: Greenlet): """ Ensures an error on the passed greenlet crashes self/main greenlet. """ def remove(_): self.greenlets.remove(greenlet) self.greenlets.append(greenlet) greenlet.link_exception(self.on_error) greenlet.link_value(remove)
def _async_incoming_msg_handler(self, transport_session, message, exception_handler): greenlet = Greenlet( self._incoming_msg_handler, transport_session, message, ) greenlet.link_exception(exception_handler) self._msg_handler_pool.start(greenlet, blocking=True)
def start(self): greenlet = Greenlet(self._start) greenlet.link_exception(self._logGreenletError) TIMER = gevent.greenlet.Greenlet(self._timer) TIMER.start() # Start and wait until the log server stops (main greenlet). greenlet.start() greenlet.join()
def __make(self, is_greedy=False): # type: (bool) -> Greenlet if self._no_self: g = Greenlet(self._fn, *self._fn_arg, **self._fn_kw) else: g = Greenlet(self._fn, self, *self._fn_arg, **self._fn_kw) if not is_greedy: # normal scenario g.link_value(self.__callback) g.link_exception(self.__err_callback) return g
def _schedule_new_greenlet(self, func: Callable, *args: Any, in_seconds_from_now: int = None, **kwargs: Any) -> Greenlet: """ Spawn a sub-task and ensures an error on it crashes self/main greenlet """ def on_success(greenlet: Greenlet) -> None: if greenlet in self.greenlets: self.greenlets.remove(greenlet) greenlet = Greenlet(func, *args, **kwargs) greenlet.link_exception(self.on_error) greenlet.link_value(on_success) self.greenlets.append(greenlet) if in_seconds_from_now: greenlet.start_later(in_seconds_from_now) else: greenlet.start() return greenlet
def start(self): print('Starting message server') for forwardAddress in self.forwardAddresses: client = MessageServerClient(forwardAddress) server.forwardClients.add(client) self._connect() self._storePidInPidFile() gevent.core.signal(signal.SIGHUP, self.stop) gevent.core.signal(signal.SIGINT, self.stop) gevent.core.signal(signal.SIGTERM, self.stop) greenlet = Greenlet(self.receiveMessages) greenlet.link_exception(self._logGreenletError) greenlet2 = Greenlet(self.processLogMessages) greenlet2.link_exception(self._logGreenletError) greenlet2.start() greenlet3 = Greenlet(self._timer) greenlet3.link_exception(self._logGreenletError) greenlet3.start() greenlet.start() storeLocallyStr = str(self.storeLocally) addresses = [client.address for client in self.forwardClients] addressesStr = ', '.join(addresses) print( '''\ Message server started listens on: %s stores locally: %s forwards to: %s pid: %d pid file: %s''' % (self._address, storeLocallyStr, addressesStr, self._pid, self._pidFile)) # Wait until the log server stops (main greenlet). try: greenlet.join() except KeyboardInterrupt: # Ignore this error. pass
def start(self): print('Starting message server') for forwardAddress in self.forwardAddresses: client = MessageServerClient(forwardAddress) server.forwardClients.add(client) self._connect() self._storePidInPidFile() gevent.core.signal(signal.SIGHUP, self.stop) gevent.core.signal(signal.SIGINT, self.stop) gevent.core.signal(signal.SIGTERM, self.stop) greenlet = Greenlet(self.receiveMessages) greenlet.link_exception(self._logGreenletError) greenlet2 = Greenlet(self.processLogMessages) greenlet2.link_exception(self._logGreenletError) greenlet2.start() greenlet3 = Greenlet(self._timer) greenlet3.link_exception(self._logGreenletError) greenlet3.start() greenlet.start() storeLocallyStr = str(self.storeLocally) addresses = [client.address for client in self.forwardClients] addressesStr = ', '.join(addresses) print(('''\ Message server started listens on: %s stores locally: %s forwards to: %s pid: %d pid file: %s''' % (self._address, storeLocallyStr, addressesStr, self._pid, self._pidFile))) # Wait until the log server stops (main greenlet). try: greenlet.join() except KeyboardInterrupt: # Ignore this error. pass
def add_pending_greenlet(self, greenlet: gevent.Greenlet): greenlet.link_exception(self.on_error)
def add(self, task_name: str, greenlet: gevent.Greenlet) -> None: greenlet.link_exception(self._handle_killed_greenlets) greenlet.task_name = task_name self.greenlets.append(greenlet)
def add(self, task_name: str, greenlet: gevent.Greenlet, exception_is_error: bool) -> None: greenlet.link_exception(self._handle_killed_greenlets) greenlet.task_name = task_name greenlet.exception_is_error = exception_is_error self.greenlets.append(greenlet)