def _client_loop(self): try: backoff = Backoff(start=0.1, limit=10, rate=5) while True: self.logger.info("Starting new irc connection") client = girc.Client(**self.irc_kwargs) self.logger.debug("Joining channels: {}".format(self.all_open_channels)) for channel in self.all_open_channels: client.channel(channel).join() client.handler(self._client_recv, command=girc.message.Privmsg) self._client.set(client) try: client.start() self.logger.debug("Started new irc connection") backoff.reset() client.wait_for_stop() except Exception as ex: self.logger.warning("irc connection died, retrying in {}".format(backoff.peek()), exc_info=True) # clear _client if no-one else has if self._client.ready(): assert self._client.get() is client self._client = AsyncResult() gevent.sleep(backoff.get()) else: self.logger.info("irc connection exited gracefully, stopping") self.stop() # graceful exit return except Exception as ex: self.stop(ex)
class ClientManager(gevent.Greenlet): """Wrapper for a client to manage clean restarts, etc""" # We mostly handle state via a synchronous main function, hence we base off Greenlet INIT_ARGS = {'hostname', 'nick', 'port', 'password', 'ident', 'real_name', 'twitch'} client = None _can_signal = False # indicates if main loop is in good state to get a stop/restart _stop = False # indicates to quit after next client quit class _Restart(Exception): """Indicates the client manager should cleanly disconnect and reconnect""" def __init__(self, name, handoff_data=None): self.name = name self.handoff_data = handoff_data self.logger = main_logger.getChild(name) super(ClientManager, self).__init__() def stop(self, message): """Gracefully stop the client""" self._stop = True if self._can_signal: self.client.quit("Shutting down", block=False) else: # we are mid-restart or similar, just kill the main loop self.kill(block=False) def restart(self, message): """Gracefully restart the client""" # if can_signal is false, restarting isn't a valid operation (ie. we're already restarting) # otherwise, send a _Restart exception to the main loop if self._can_signal: self.kill(self._Restart(message), block=False) def handoff(self): """Gracefully shut down and prepare for handoff. This stops the client and returns a dict suitable to pass as config.handoff_data[name] to a child or re-exec()ed process. However, if the client is not currently in a good state for handoff (eg. it is currently restarting) this method will still stop the client manager, but will return None. In this case, there was no state to handoff so the best thing to do is let the child re-create a new client. """ # Note this method intentionally leaks an fd so we can't accidentially close it # due to destructors. This fd is then passed onto the child / re-exec()ed process. self.logger.info("Attempting to handoff") if not self._can_signal: self.logger.info("Handoff aborted - client is not running") # we are mid-restart or similar, just kill the main loop self.kill(block=False) return try: self.client._prepare_for_handoff() except Exception: # this can happen if we're mid-start, best thing to do is just abort self.logger.info("Handoff aborted - client in bad state") self.client.stop() self.kill(block=False) return data = self.client._get_handoff_data() data['fd'] = os.dup(self.client._socket.fileno()) self.logger.info("Handoff initiated with data {!r}".format(data)) # this will gracefully stop, which will cause the main loop to exit self.client._finalize_handoff() return data def _parse_config_plugins(self): plugins = [] for name in config.clients_with_defaults[self.name].get('plugins', []): args = () if ':' in name: name, args = name.split(':', 1) args = args.split(',') plugins.append((name, args)) return plugins def _run(self): if self.name in clients: return # already running, ignore second attempt to start clients[self.name] = self try: self.retry_timer = Backoff(RETRY_START, RETRY_LIMIT, RETRY_FACTOR) while not self._stop: if self.name not in config.clients_with_defaults: raise Exception("No such client {!r}".format(self.name)) options = config.clients_with_defaults[self.name] channels = options.get('channels', []) plugins = self._parse_config_plugins() try: if self.handoff_data: self.logger.info("Accepting handoff with data {!r}".format(self.handoff_data)) client_sock = socket.fromfd(self.handoff_data.pop('fd'), socket.AF_INET, socket.SOCK_STREAM) self.client = EkimbotClient._from_handoff(client_sock, name=self.name, logger=self.logger, **self.handoff_data) self.handoff_data = None else: self.logger.info("Starting client") self.client = EkimbotClient(self.name, logger=self.logger, **{key: options[key] for key in self.INIT_ARGS if key in options}) self.logger.info("Enabling {} plugins".format(len(plugins))) for plugin, args in plugins: self.logger.debug("Enabling plugin {} with args {}".format(plugin, args)) ClientPlugin.enable(plugin, self.client, *args) self.logger.info("Joining {} channels".format(len(channels))) for channel in channels: self.logger.debug("Joining channel {}".format(channel)) self.client.channel(channel).join() try: self._can_signal = True self.client.start() self.logger.debug("Client started") self.retry_timer.reset() self.client.wait_for_stop() self.logger.info("Client exited cleanly, not re-connecting") break finally: self._can_signal = False except Exception as ex: if isinstance(ex, self._Restart): self.logger.info("Client gracefully restarting: {}".format(ex)) try: self.client.quit(str(ex)) except Exception: self.logger.warning("Client failed during graceful restart", exc_info=True) else: self.logger.warning("Client failed, re-connecting in {}s".format(self.retry_timer.peek()), exc_info=True) if not self._stop and not isinstance(ex, self._Restart): gevent.sleep(self.retry_timer.get()) except Exception: self.logger.critical("run_client failed with unhandled exception") raise finally: assert clients[self.name] is self del clients[self.name]