def _client_loop(self):
try:
backoff = Backoff(start=0.1, limit=10, rate=5)
while True:
self.logger.info("Starting new irc connection")
client = girc.Client(**self.irc_kwargs)
self.logger.debug("Joining channels: {}".format(self.all_open_channels))
for channel in self.all_open_channels:
client.channel(channel).join()
client.handler(self._client_recv, command=girc.message.Privmsg)
self._client.set(client)
try:
client.start()
self.logger.debug("Started new irc connection")
backoff.reset()
client.wait_for_stop()
except Exception as ex:
self.logger.warning("irc connection died, retrying in {}".format(backoff.peek()), exc_info=True)
# clear _client if no-one else has
if self._client.ready():
assert self._client.get() is client
self._client = AsyncResult()
gevent.sleep(backoff.get())
else:
self.logger.info("irc connection exited gracefully, stopping")
self.stop() # graceful exit
return
except Exception as ex:
self.stop(ex)
class ClientManager(gevent.Greenlet):
"""Wrapper for a client to manage clean restarts, etc"""
# We mostly handle state via a synchronous main function, hence we base off Greenlet
INIT_ARGS = {'hostname', 'nick', 'port', 'password', 'ident', 'real_name', 'twitch'}
client = None
_can_signal = False # indicates if main loop is in good state to get a stop/restart
_stop = False # indicates to quit after next client quit
class _Restart(Exception):
"""Indicates the client manager should cleanly disconnect and reconnect"""
def __init__(self, name, handoff_data=None):
self.name = name
self.handoff_data = handoff_data
self.logger = main_logger.getChild(name)
super(ClientManager, self).__init__()
def stop(self, message):
"""Gracefully stop the client"""
self._stop = True
if self._can_signal:
self.client.quit("Shutting down", block=False)
else:
# we are mid-restart or similar, just kill the main loop
self.kill(block=False)
def restart(self, message):
"""Gracefully restart the client"""
# if can_signal is false, restarting isn't a valid operation (ie. we're already restarting)
# otherwise, send a _Restart exception to the main loop
if self._can_signal:
self.kill(self._Restart(message), block=False)
def handoff(self):
"""Gracefully shut down and prepare for handoff.
This stops the client and returns a dict suitable to pass as config.handoff_data[name]
to a child or re-exec()ed process.
However, if the client is not currently in a good state for handoff (eg. it is currently restarting)
this method will still stop the client manager, but will return None. In this case,
there was no state to handoff so the best thing to do is let the child re-create a new client.
"""
# Note this method intentionally leaks an fd so we can't accidentially close it
# due to destructors. This fd is then passed onto the child / re-exec()ed process.
self.logger.info("Attempting to handoff")
if not self._can_signal:
self.logger.info("Handoff aborted - client is not running")
# we are mid-restart or similar, just kill the main loop
self.kill(block=False)
return
try:
self.client._prepare_for_handoff()
except Exception:
# this can happen if we're mid-start, best thing to do is just abort
self.logger.info("Handoff aborted - client in bad state")
self.client.stop()
self.kill(block=False)
return
data = self.client._get_handoff_data()
data['fd'] = os.dup(self.client._socket.fileno())
self.logger.info("Handoff initiated with data {!r}".format(data))
# this will gracefully stop, which will cause the main loop to exit
self.client._finalize_handoff()
return data
def _parse_config_plugins(self):
plugins = []
for name in config.clients_with_defaults[self.name].get('plugins', []):
args = ()
if ':' in name:
name, args = name.split(':', 1)
args = args.split(',')
plugins.append((name, args))
return plugins
def _run(self):
if self.name in clients:
return # already running, ignore second attempt to start
clients[self.name] = self
try:
self.retry_timer = Backoff(RETRY_START, RETRY_LIMIT, RETRY_FACTOR)
while not self._stop:
if self.name not in config.clients_with_defaults:
raise Exception("No such client {!r}".format(self.name))
options = config.clients_with_defaults[self.name]
channels = options.get('channels', [])
plugins = self._parse_config_plugins()
try:
if self.handoff_data:
self.logger.info("Accepting handoff with data {!r}".format(self.handoff_data))
client_sock = socket.fromfd(self.handoff_data.pop('fd'), socket.AF_INET, socket.SOCK_STREAM)
self.client = EkimbotClient._from_handoff(client_sock, name=self.name, logger=self.logger, **self.handoff_data)
self.handoff_data = None
else:
self.logger.info("Starting client")
self.client = EkimbotClient(self.name,
logger=self.logger,
**{key: options[key] for key in self.INIT_ARGS if key in options})
self.logger.info("Enabling {} plugins".format(len(plugins)))
for plugin, args in plugins:
self.logger.debug("Enabling plugin {} with args {}".format(plugin, args))
ClientPlugin.enable(plugin, self.client, *args)
self.logger.info("Joining {} channels".format(len(channels)))
for channel in channels:
self.logger.debug("Joining channel {}".format(channel))
self.client.channel(channel).join()
try:
self._can_signal = True
self.client.start()
self.logger.debug("Client started")
self.retry_timer.reset()
self.client.wait_for_stop()
self.logger.info("Client exited cleanly, not re-connecting")
break
finally:
self._can_signal = False
except Exception as ex:
if isinstance(ex, self._Restart):
self.logger.info("Client gracefully restarting: {}".format(ex))
try:
self.client.quit(str(ex))
except Exception:
self.logger.warning("Client failed during graceful restart", exc_info=True)
else:
self.logger.warning("Client failed, re-connecting in {}s".format(self.retry_timer.peek()), exc_info=True)
if not self._stop and not isinstance(ex, self._Restart):
gevent.sleep(self.retry_timer.get())
except Exception:
self.logger.critical("run_client failed with unhandled exception")
raise
finally:
assert clients[self.name] is self
del clients[self.name]
