class Watchdog(object):
"""
Runs a callable in a gevent greenlet, and restarts the greenlet with the
same callable iff any exception is raised from the greenlet. Uses
exponential backoff to respawn the greenlet until eventually it gives up.
"""
def __init__(self, func, backoff=None):
if not callable(func):
raise ValueError('Func argument is not callable')
if not backoff:
backoff = TimeSensitiveBackoff()
self.func = func
self.backoff = backoff
def __call__(self, greenlet):
try:
logger.exception(greenlet.exception)
time.sleep(self.backoff.next())
greenlet = self.respawn()
return greenlet
except StopIteration:
_failhard('backoff exceeded', self.func, greenlet)
get_hub().parent.throw(SystemExit())
def spawn(self):
self.greenlet = Greenlet(self.func)
self.greenlet.link_exception(self)
self.greenlet.start()
return self.greenlet
def respawn(self):
return self.spawn()
def add_pending_greenlet(self, greenlet: Greenlet):
""" Ensures an error on the passed greenlet crashes self/main greenlet. """
def remove(_):
self.greenlets.remove(greenlet)
self.greenlets.append(greenlet)
greenlet.link_exception(self.on_error)
greenlet.link_value(remove)
def _async_incoming_msg_handler(self, transport_session, message,
exception_handler):
greenlet = Greenlet(
self._incoming_msg_handler,
transport_session,
message,
)
greenlet.link_exception(exception_handler)
self._msg_handler_pool.start(greenlet, blocking=True)
def start(self):
greenlet = Greenlet(self._start)
greenlet.link_exception(self._logGreenletError)
TIMER = gevent.greenlet.Greenlet(self._timer)
TIMER.start()
# Start and wait until the log server stops (main greenlet).
greenlet.start()
greenlet.join()
def start(self):
greenlet = Greenlet(self._start)
greenlet.link_exception(self._logGreenletError)
TIMER = gevent.greenlet.Greenlet(self._timer)
TIMER.start()
# Start and wait until the log server stops (main greenlet).
greenlet.start()
greenlet.join()
def __make(self, is_greedy=False):
# type: (bool) -> Greenlet
if self._no_self:
g = Greenlet(self._fn, *self._fn_arg, **self._fn_kw)
else:
g = Greenlet(self._fn, self, *self._fn_arg, **self._fn_kw)
if not is_greedy:
# normal scenario
g.link_value(self.__callback)
g.link_exception(self.__err_callback)
return g
def _schedule_new_greenlet(self,
func: Callable,
*args: Any,
in_seconds_from_now: int = None,
**kwargs: Any) -> Greenlet:
""" Spawn a sub-task and ensures an error on it crashes self/main greenlet """
def on_success(greenlet: Greenlet) -> None:
if greenlet in self.greenlets:
self.greenlets.remove(greenlet)
greenlet = Greenlet(func, *args, **kwargs)
greenlet.link_exception(self.on_error)
greenlet.link_value(on_success)
self.greenlets.append(greenlet)
if in_seconds_from_now:
greenlet.start_later(in_seconds_from_now)
else:
greenlet.start()
return greenlet
def start(self):
print('Starting message server')
for forwardAddress in self.forwardAddresses:
client = MessageServerClient(forwardAddress)
server.forwardClients.add(client)
self._connect()
self._storePidInPidFile()
gevent.core.signal(signal.SIGHUP, self.stop)
gevent.core.signal(signal.SIGINT, self.stop)
gevent.core.signal(signal.SIGTERM, self.stop)
greenlet = Greenlet(self.receiveMessages)
greenlet.link_exception(self._logGreenletError)
greenlet2 = Greenlet(self.processLogMessages)
greenlet2.link_exception(self._logGreenletError)
greenlet2.start()
greenlet3 = Greenlet(self._timer)
greenlet3.link_exception(self._logGreenletError)
greenlet3.start()
greenlet.start()
storeLocallyStr = str(self.storeLocally)
addresses = [client.address for client in self.forwardClients]
addressesStr = ', '.join(addresses)
print(
'''\
Message server started
listens on: %s
stores locally: %s
forwards to: %s
pid: %d
pid file: %s''' % (self._address, storeLocallyStr, addressesStr, self._pid,
self._pidFile))
# Wait until the log server stops (main greenlet).
try:
greenlet.join()
except KeyboardInterrupt:
# Ignore this error.
pass
def start(self):
print('Starting message server')
for forwardAddress in self.forwardAddresses:
client = MessageServerClient(forwardAddress)
server.forwardClients.add(client)
self._connect()
self._storePidInPidFile()
gevent.core.signal(signal.SIGHUP, self.stop)
gevent.core.signal(signal.SIGINT, self.stop)
gevent.core.signal(signal.SIGTERM, self.stop)
greenlet = Greenlet(self.receiveMessages)
greenlet.link_exception(self._logGreenletError)
greenlet2 = Greenlet(self.processLogMessages)
greenlet2.link_exception(self._logGreenletError)
greenlet2.start()
greenlet3 = Greenlet(self._timer)
greenlet3.link_exception(self._logGreenletError)
greenlet3.start()
greenlet.start()
storeLocallyStr = str(self.storeLocally)
addresses = [client.address for client in self.forwardClients]
addressesStr = ', '.join(addresses)
print(('''\
Message server started
listens on: %s
stores locally: %s
forwards to: %s
pid: %d
pid file: %s''' % (self._address, storeLocallyStr, addressesStr, self._pid, self._pidFile)))
# Wait until the log server stops (main greenlet).
try:
greenlet.join()
except KeyboardInterrupt:
# Ignore this error.
pass
def add_pending_greenlet(self, greenlet: gevent.Greenlet):
greenlet.link_exception(self.on_error)
def add_pending_greenlet(self, greenlet: gevent.Greenlet):
greenlet.link_exception(self.on_error)
def add(self, task_name: str, greenlet: gevent.Greenlet) -> None:
greenlet.link_exception(self._handle_killed_greenlets)
greenlet.task_name = task_name
self.greenlets.append(greenlet)
def add(self, task_name: str, greenlet: gevent.Greenlet,
exception_is_error: bool) -> None:
greenlet.link_exception(self._handle_killed_greenlets)
greenlet.task_name = task_name
greenlet.exception_is_error = exception_is_error
self.greenlets.append(greenlet)
