def look_after(self): """Checks for the watchdoged watchers and restart a process if no received watchdog after the loop_rate * max_count period. """ # if first check, do a full discovery first. if self.starting: self._discover_monitored_pids() self.starting = False max_timeout = self.loop_rate * self.max_count too_old_time = time.time() - max_timeout for pid, detail in self.pid_status.items(): if detail['last_activity'] < too_old_time: logger.info("watcher:%s, pid:%s is not responding. Kill it !", detail['watcher'], pid) props = dict(name=detail['watcher'], pid=int(pid)) if self.stop_signal is not None: props['signum'] = self.stop_signal if self.graceful_timeout is not None: props['graceful_timeout'] = self.graceful_timeout self.cast('kill', **props) # Trusting watcher to eventually stop the process after # graceful timeout del self.pid_status[pid]
def handle_recv(self, data): """Handle received message from circusd We need to handle two messages: - spawn: add a new monitored child pid - reap: remove a killed child pid from monitoring """ watcher_name, action, msg = self.split_data(data) logger.debug("received data from circusd: watcher.%s.%s, %s", watcher_name, action, msg) # check if monitored watchers: if self._match_watcher_name(watcher_name): try: message = self.load_message(msg) except ValueError: logger.error("Error while decoding json for message: %s", msg) else: if "process_pid" not in message: logger.warning('no process_pid in message') return pid = str(message.get("process_pid")) if action == "spawn": self.pid_status[pid] = dict(watcher=watcher_name, last_activity=time.time()) logger.info("added new monitored pid for %s:%s", watcher_name, pid) # very questionable fix for Py3 here! # had to add check for pid in self.pid_status elif action == "reap" and pid in self.pid_status: old_pid = self.pid_status.pop(pid) logger.info("removed monitored pid for %s:%s", old_pid['watcher'], pid)
def start(self): """Start. """ if not self.stopped: return if self.on_demand and not self.arbiter.socket_event: return self.stopped = False if not self.call_hook("before_start"): logger.debug("Aborting startup") self.stopped = True return False self._create_redirectors() self.reap_processes() self.spawn_processes() if not self.call_hook("after_start"): logger.debug("Aborting startup") self.stop() return False if self.stdout_redirector is not None: self.stdout_redirector.start() if self.stderr_redirector is not None: self.stderr_redirector.start() logger.info("%s started" % self.name) self.notify_event("start", {"time": time.time()}) return True
def start(self): """Starts all the watchers. If the ioloop has been provided during __init__() call, starts all watchers as a standard coroutine If the ioloop hasn't been provided during __init__() call (default), starts all watchers and the eventloop (and blocks here). In this mode the method MUST NOT yield anything because it's called as a standard method. """ logger.info("Starting master on pid %s", self.pid) self.initialize() # start controller self.ctrl.start() self._restarting = False try: # initialize processes logger.debug('Initializing watchers') if self._provided_loop: yield self.start_watchers() else: # start_watchers will be called just after the start_io_loop() self.loop.add_future(self.start_watchers(), lambda x: None) logger.info('Arbiter now waiting for commands') self._running = True if not self._provided_loop: # If an event loop is not provided, block at this line self.start_io_loop() finally: if not self._provided_loop: # If an event loop is not provided, do some cleaning self.stop_controller_and_close_sockets() raise gen.Return(self._restarting)
def _stop(self, close_output_streams=False, for_shutdown=False): if self.is_stopped(): return self._status = "stopping" skip = for_shutdown and self.use_papa if not skip: logger.debug('stopping the %s watcher' % self.name) logger.debug('gracefully stopping processes [%s] for %ss' % (self.name, self.graceful_timeout)) # We ignore the hook result self.call_hook('before_stop') yield self.kill_processes() self.reap_processes() # stop redirectors if self.stream_redirector: self.stream_redirector.stop() self.stream_redirector = None if close_output_streams: if self.stdout_stream and hasattr(self.stdout_stream, 'close'): self.stdout_stream.close() if self.stderr_stream and hasattr(self.stderr_stream, 'close'): self.stderr_stream.close() # notify about the stop if skip: logger.info('%s left running in papa', self.name) else: if self.evpub_socket is not None: self.notify_event("stop", {"time": time.time()}) self._status = "stopped" # We ignore the hook result self.call_hook('after_stop') logger.info('%s stopped', self.name)
def stop(self): """Stop. """ logger.debug('stopping the %s watcher' % self.name) # stop redirectors if self.stdout_redirector is not None: self.stdout_redirector.stop() self.stdout_redirector = None if self.stderr_redirector is not None: self.stderr_redirector.stop() self.stderr_redirector = None limit = time.time() + self.graceful_timeout logger.debug('gracefully stopping processes [%s] for %ss' % (self.name, self.graceful_timeout)) # We ignore the hook result self.call_hook('before_stop') while self.get_active_processes() and time.time() < limit: self.kill_processes(signal.SIGTERM) self.reap_processes() self.kill_processes(signal.SIGKILL) if self.evpub_socket is not None: self.notify_event("stop", {"time": time.time()}) self.stopped = True # We ignore the hook result self.call_hook('after_stop') logger.info('%s stopped', self.name)
def start(self): """Starts all the watchers. The start command is an infinite loop that waits for any command from a client and that watches all the processes and restarts them if needed. """ logger.info("Starting master on pid %s", self.pid) self.initialize() # start controller self.ctrl.start() # initialize processes logger.debug("Initializing watchers") for watcher in self.iter_watchers(): watcher.start() logger.info("Arbiter now waiting for commands") while True: try: self.loop.start() except zmq.ZMQError as e: if e.errno == errno.EINTR: continue else: raise else: break self.ctrl.stop() self.evpub_socket.close()
def _stop(self, close_output_streams=False, for_shutdown=False): if self.is_stopped(): return self._status = "stopping" skip = for_shutdown and self.use_papa if not skip: logger.debug('stopping the %s watcher' % self.name) logger.debug('gracefully stopping processes [%s] for %ss' % ( self.name, self.graceful_timeout)) # We ignore the hook result self.call_hook('before_stop') yield self.kill_processes() self.reap_processes() # stop redirectors if self.stream_redirector: self.stream_redirector.stop() self.stream_redirector = None if close_output_streams: if self.stdout_stream and hasattr(self.stdout_stream, 'close'): self.stdout_stream.close() if self.stderr_stream and hasattr(self.stderr_stream, 'close'): self.stderr_stream.close() # notify about the stop if skip: logger.info('%s left running in papa', self.name) else: if self.evpub_socket is not None: self.notify_event("stop", {"time": time.time()}) self._status = "stopped" # We ignore the hook result self.call_hook('after_stop') logger.info('%s stopped', self.name)
def stop(self): """Stop. """ logger.debug('stopping the %s watcher' % self.name) # stop redirectors if self.stdout_redirector is not None: self.stdout_redirector.kill() if self.stderr_redirector is not None: self.stderr_redirector.kill() limit = time.time() + self.graceful_timeout logger.debug('gracefully stopping processes [%s] for %ss' % ( self.name, self.graceful_timeout)) while self.get_active_processes() and time.time() < limit: self.kill_processes(signal.SIGTERM) try: time.sleep(0.1) except KeyboardInterrupt: pass self.reap_processes() self.kill_processes(signal.SIGKILL) if self.evpub_socket is not None: self.notify_event("stop", {"time": time.time()}) self.stopped = True logger.info('%s stopped', self.name)
def restart(self): """Restart. """ self.notify_event("restart", {"time": time.time()}) self.stop() self.start() logger.info('%s restarted', self.name)
def spawn_fly(self): if self.stopped: return self._fly_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: fly = None try: fly = Fly(self._fly_counter, self.cmd, wdir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env) self.flies[self._fly_counter] = fly logger.info('running %s fly [pid %d]' % (self.name, fly.pid)) except OSError, e: logger.warning('error in %r: %s' % (self.name, str(e))) if fly is None: nb_tries += 1 continue else: return
def restart(self): """ restart """ self.send_msg("restart", {"time": time.time()}) self.stop() self.start() logger.info('%s restarted' % self.name)
def restart(self): """Restart. """ self.send_msg("restart", {"time": time.time()}) self.stop() self.start() logger.info("%s restarted", self.name)
def stop(self): """Stop. """ logger.debug('stopping the %s watcher' % self.name) # stop redirectors if self.stdout_redirector is not None: self.stdout_redirector.kill() if self.stderr_redirector is not None: self.stderr_redirector.kill() limit = time.time() + self.graceful_timeout logger.debug('gracefully stopping processes [%s] for %ss' % ( self.name, self.graceful_timeout)) # We ignore the hook result self.call_hook('before_stop') while self.get_active_processes() and time.time() < limit: self.kill_processes(signal.SIGTERM) self.reap_processes() self.kill_processes(signal.SIGKILL) if self.evpub_socket is not None: self.notify_event("stop", {"time": time.time()}) self.stopped = True # We ignore the hook result self.call_hook('after_stop') logger.info('%s stopped', self.name)
def start(self): """Start. """ if not self.stopped: return self.stopped = False if not self.call_hook('before_start'): logger.debug('Aborting startup') self.stopped = True return False self._create_redirectors() self.reap_processes() self.spawn_processes() if not self.call_hook('after_start'): logger.debug('Aborting startup') self.stop() return False if self.stdout_redirector is not None: self.stdout_redirector.start() if self.stderr_redirector is not None: self.stderr_redirector.start() logger.info('%s started' % self.name) self.notify_event("start", {"time": time.time()}) return True
def _reload(self, graceful=True, sequential=False): """ reload """ if not graceful and sequential: logger.warn("with graceful=False, sequential=True is ignored") if self.prereload_fn is not None: self.prereload_fn(self) if not graceful: yield self._restart() return if self.is_stopped(): yield self._start() elif self.send_hup: for process in self.processes.values(): logger.info("SENDING HUP to %s" % process.pid) process.send_signal(signal.SIGHUP) else: if sequential: active_processes = self.get_active_processes() for process in active_processes: yield self.kill_process(process) self.reap_process(process.pid) self.spawn_process() yield tornado_sleep(self.warmup_delay) else: for i in range(self.numprocesses): self.spawn_process() yield self.manage_processes() self.notify_event("reload", {"time": time.time()}) logger.info('%s reloaded', self.name)
def start(self): """Starts all the watchers. If the ioloop has been provided during __init__() call, starts all watchers as a standard coroutine If the ioloop hasn't been provided during __init__() call (default), starts all watchers and the eventloop (and blocks here). In this mode the method MUST NOT yield anything because it's called as a standard method. """ logger.info("Starting master on pid %s", self.pid) self.initialize() # start controller self.ctrl.start() self._restarting = False try: # initialize processes logger.debug('Initializing watchers') if self._provided_loop: yield self.start_watchers() else: # start_watchers will be called just after the start_io_loop() self.loop.add_future(self.start_watchers(), lambda x: None) logger.info('Arbiter now waiting for commands') if not self._provided_loop: # If an event loop is not provided, block at this line self.start_io_loop() finally: if not self._provided_loop: # If an event loop is not provided, do some cleaning self.stop_controller_and_close_sockets() raise gen.Return(self._restarting)
def stop(self): """Stop. """ logger.debug("stopping the %s watcher" % self.name) # stop redirectors if self.stdout_redirector is not None: self.stdout_redirector.kill() if self.stderr_redirector is not None: self.stderr_redirector.kill() limit = time.time() + self.graceful_timeout logger.debug("gracefully stopping processes [%s] for %ss" % (self.name, self.graceful_timeout)) while self.get_active_pids() and time.time() < limit: self.kill_processes(signal.SIGTERM) time.sleep(0.1) self.reap_processes() self.kill_processes(signal.SIGKILL) if self.evpub_socket is not None: self.send_msg("stop", {"time": time.time()}) self.stopped = True logger.info("%s stopped", self.name)
def spawn_fly(self): """ spawn fly """ if self.stopped: return self._fly_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: fly = None try: fly = Fly(self._fly_counter, self.cmd, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env) self.flies[self._fly_counter] = fly logger.info('running %s fly [pid %d]' % (self.name, fly.pid)) except OSError, e: logger.warning('error in %r: %s' % (self.name, str(e))) if fly is None: nb_tries += 1 continue else: self.send_msg("spawn", {"fly_id": fly.wid, "fly_pid": fly.pid, "time": time.time()}) return
def look_after(self): # Get concerned watchers # Note: watchers and their configuration can change at any time watchers = self.call('list')['watchers'] if self.watchers != '*': watchers = [w for w in watchers if w in self.watchers.split(',')] for watcher in list(self.infos.keys()): if watcher not in watchers: del self.infos[watcher] # Clean obsolete watchers # Check modifications for w in watchers: # We make sure not to reload to often (wait at least 2secs) if self.infos[w].get('last_restart') < time.time() - 2: w_info = self.call('get', name=w, keys=['cmd', 'working_dir']) previous_path = self.infos[w].get('path') if self.use_working_dir: current_path = w_info['options']['working_dir'] else: current_path = w_info['options']['cmd'] self.infos[w]['path'] = current_path self.infos[w]['mtime'] = self.mtime_of_path(current_path) if self.is_modified(w, previous_path): if self.use_reload: logger.info('%s modified. Reloading.', self.infos[w]['path']) self.call('reload', name=w) else: logger.info('%s modified. Reloading.', self.infos[w]['path']) self.call('restart', name=w) self.infos[w]['last_restart'] = time.time()
def start(self): logger.info('Starting the stats streamer') self._init() logger.debug('Initial list is ' + str(self._pids)) self.running = True self.stats.start() self.publisher.start() logger.debug('Now looping to get circusd events') while self.running: try: self.loop.start() except zmq.ZMQError as e: logger.debug(str(e)) if e.errno == errno.EINTR: continue elif e.errno == zmq.ETERM: break else: logger.debug("got an unexpected error %s (%s)", str(e), e.errno) raise else: break self.sub_socket.close()
def look_after(self): """Checks for the watchdoged watchers and restart a process if no received watchdog after the loop_rate * max_count period. """ # if first check, do a full discovery first. if self.starting: self._discover_monitored_pids() self.starting = False max_timeout = self.loop_rate * self.max_count too_old_time = time.time() - max_timeout for pid, detail in self.pid_status.items(): if detail["last_activity"] < too_old_time: logger.info("watcher:%s, pid:%s is not responding. Kill it !", detail["watcher"], pid) props = dict(name=detail["watcher"], pid=int(pid)) if self.stop_signal is not None: props["signum"] = self.stop_signal if self.graceful_timeout is not None: props["graceful_timeout"] = self.graceful_timeout self.cast("kill", **props) # Trusting watcher to eventually stop the process after # graceful timeout del self.pid_status[pid]
def stop(self): """Stop. """ logger.debug('stopping the %s watcher' % self.name) # stop redirectors if self.stdout_redirector is not None: self.stdout_redirector.kill() if self.stderr_redirector is not None: self.stderr_redirector.kill() limit = time.time() + self.graceful_timeout logger.debug('gracefully stopping processes [%s] for %ss' % (self.name, self.graceful_timeout)) while self.get_active_processes() and time.time() < limit: self.kill_processes(signal.SIGTERM) try: time.sleep(0.1) except KeyboardInterrupt: pass self.reap_processes() self.kill_processes(signal.SIGKILL) if self.evpub_socket is not None: self.notify_event("stop", {"time": time.time()}) self.stopped = True logger.info('%s stopped', self.name)
def start(self): """Starts all the watchers. The start command is an infinite loop that waits for any command from a client and that watches all the processes and restarts them if needed. """ logger.info("Starting master on pid %s", self.pid) self.initialize() # start controller self.ctrl.start() # initialize processes logger.debug('Initializing watchers') for watcher in self.iter_watchers(): watcher.start() logger.info('Arbiter now waiting for commands') while True: try: self.loop.start() except zmq.ZMQError as e: if e.errno == errno.EINTR: continue else: raise else: break self.ctrl.stop() self.evpub_socket.close()
def __init__(self, controller): self.controller = controller # init signals logger.info('Registering signals...') self._old = {} self._register()
def handle_recv(self, data): """Handle received message from circusd We need to handle two messages: - spawn: add a new monitored child pid - reap: remove a killed child pid from monitoring """ topic, msg = data topic_parts = topic.split(".") logger.debug("received data from circusd: %s, %s", topic_parts, msg) # check if monitored watchers: if (topic_parts[0] == 'watcher' and self._match_watcher_name(topic_parts[1])): try: message = json.loads(msg) except ValueError: logger.error("Error while decoding json for message: %s", msg) else: if "process_pid" not in message: logger.warning('no process_pid in message') return pid = str(message.get("process_pid")) if topic_parts[2] == "spawn": self.pid_status[pid] = dict(watcher=topic_parts[1], last_activity=time.time()) logger.info("added new monitored pid for %s:%s", topic_parts[1], pid) elif topic_parts[2] == "reap": old_pid = self.pid_status.pop(pid) logger.info("removed monitored pid for %s:%s", old_pid['watcher'], pid)
def restart(self): """Restart. """ self.send_msg("restart", {"time": time.time()}) self.stop() self.start() logger.info('%s restarted', self.name)
def _start(self): """Start. """ if not self.is_stopped(): return if self.on_demand and not self.arbiter.socket_event: return if not self.call_hook('before_start'): logger.debug('Aborting startup') return self._status = "starting" self._create_redirectors() self.reap_processes() yield self.spawn_processes() if not self.call_hook('after_start'): logger.debug('Aborting startup') yield self._stop() return if self.stdout_redirector is not None: self.stdout_redirector.start() if self.stderr_redirector is not None: self.stderr_redirector.start() self._status = "active" logger.info('%s started' % self.name) self.notify_event("start", {"time": time.time()})
def start(self): """Start. """ if not self.stopped: return if self.on_demand and not self.arbiter.socket_event: return self.stopped = False if not self.call_hook('before_start'): logger.debug('Aborting startup') self.stopped = True return False self._create_redirectors() self.reap_processes() self.spawn_processes() if not self.call_hook('after_start'): logger.debug('Aborting startup') self.stop() return False if self.stdout_redirector is not None: self.stdout_redirector.start() if self.stderr_redirector is not None: self.stderr_redirector.start() logger.info('%s started' % self.name) self.notify_event("start", {"time": time.time()}) return True
def check(self, watcher_name): timeline = self.timelines[watcher_name] if watcher_name in self.configs: conf = self.configs[watcher_name] else: conf = self.update_conf(watcher_name) tries = self.tries.get(watcher_name, 0) if len(timeline) == conf["times"]: duration = timeline[-1] - timeline[0] - self.check_delay if duration <= conf["within"]: if tries < conf["max_retry"]: logger.info("%s: flapping detected: retry in %2ds", watcher_name, conf["retry_in"]) self.cast(make_message("stop", name=watcher_name)) self.timelines[watcher_name] = [] self.tries[watcher_name] = tries + 1 def _start(): self.cast(make_message("start", name=watcher_name)) timer = Timer(conf["retry_in"], _start) timer.start() self.timers[watcher_name] = timer else: logger.info("%s: flapping detected: max retry limit", watcher_name) self.timelines[watcher_name] = [] self.tries[watcher_name] = 0 self.cast(make_message("stop", name=watcher_name)) else: self.timelines[watcher_name] = [] self.tries[watcher_name] = 0
def main(): parser = argparse.ArgumentParser(description="Runs a plugin.") parser.add_argument("--endpoint", help="The circusd ZeroMQ socket to connect to", default=DEFAULT_ENDPOINT_DEALER) parser.add_argument( "--pubsub", help="The circusd ZeroMQ pub/sub socket to connect to", default=DEFAULT_ENDPOINT_SUB ) parser.add_argument("--config", help="The plugin configuration", default=None) parser.add_argument("--version", action="store_true", default=False, help="Displays Circus version and exits.") parser.add_argument("--check-delay", type=float, default=5.0, help="Checck delay.") parser.add_argument("plugin", help="Fully qualified name of the plugin class.", nargs="?") parser.add_argument("--log-level", dest="loglevel", default="info", help="log level") parser.add_argument("--log-output", dest="logoutput", default="-", help="log output") parser.add_argument("--ssh", default=None, help="SSH Server") args = parser.parse_args() if args.version: print(__version__) sys.exit(0) if args.plugin is None: parser.print_usage() sys.exit(0) # configure the logger loglevel = LOG_LEVELS.get(args.loglevel.lower(), logging.INFO) logger.setLevel(loglevel) if args.logoutput == "-": h = logging.StreamHandler() else: h = logging.handlers.WatchedFileHandler(args.logoutput) close_on_exec(h.stream.fileno()) fmt = logging.Formatter(LOG_FMT, LOG_DATE_FMT) h.setFormatter(fmt) logger.addHandler(h) # load the plugin and run it. logger.info("Loading the plugin...") logger.info("Endpoint: %r" % args.endpoint) logger.info("Pub/sub: %r" % args.pubsub) plugin = resolve_name(args.plugin)(args.endpoint, args.pubsub, args.check_delay, args.ssh, **_str2cfg(args.config)) logger.info("Starting") try: plugin.start() except KeyboardInterrupt: pass finally: logger.info("Stopping") plugin.stop() sys.exit(0)
def start(self): if not self.stopped: return self.stopped = False self.reap_flies() self.manage_flies() logger.info('%s started' % self.name)
def _stop(self): logger.info('Arbiter exiting') self._stopping = True yield self._stop_watchers(close_output_streams=True) if self._provided_loop: cb = self.stop_controller_and_close_sockets self.loop.add_callback(cb) else: self.loop.add_timeout(time.time() + 1, self._stop_cb)
def restart(self): """Restart. """ self.notify_event("restart", {"time": time.time()}) self.stop() if self.start(): logger.info('%s restarted', self.name) else: logger.info('Failed to restart %s', self.name)
def stop_watchers(self, graceful=True, stop_alive=False): if stop_alive: logger.info("Arbiter exiting") if not self.alive: return self.alive = False for watcher in self.watchers: watcher.stop(graceful=graceful)
def _ensure_ioloop(self): if self.loop is None: self.loop = ioloop.IOLoop.current() def handle_callback_exception(callback): logger.error("Exception in callback %r", callback, exc_info=True) if not hasattr(self.loop, 'handle_callback_exception'): logger.info('Installing handle_callback_exception to loop') setattr(self.loop, 'handle_callback_exception', handle_callback_exception)
def handle_reload(self, *args): if self.send_hup: for wid, fly in self.flies.items(): logger.info("SEND HUP to %s [%s]" % (wid, fly.pid)) fly.send_signal(signal.SIGHUP) else: for i in range(self.num_flies): self.spawn_fly() self.manage_flies() return "ok"
def stop_watchers(self, stop_alive=False): if not self.alive: return if stop_alive: logger.info('Arbiter exiting') self.alive = False for watcher in self.iter_watchers(): watcher.stop()
def stop(self): # stop all the periodic callbacks running for callback in self._callbacks.values(): callback.stop() self.loop.stop() self.ctx.destroy(0) self.publisher.stop() self.stopped = True self.running = False logger.info('Stats streamer stopped')
def start(self): """Start. """ if not self.stopped: return self.stopped = False self.reap_processes() self.manage_processes() logger.info('%s started' % self.name) self.send_msg("start", {"time": time.time()})
def __stop(self, for_shutdown=False): logger.info('Arbiter exiting') self._stopping = True yield self._stop_watchers(close_output_streams=True, for_shutdown=for_shutdown) if self._provided_loop: cb = self.stop_controller_and_close_sockets self.loop.add_callback(cb) else: # stop_controller_and_close_sockets will be # called in the end of start() method self.loop.add_callback(self.loop.stop)
def __init__(self, shows, endpoint, check_delay=1., ipc_path=None): self.shows = shows self.endpoint = endpoint self.check_delay = check_delay self.ipc_path = ipc_path self.ctrl = Controller(endpoint, self, self.check_delay, self.ipc_path) self.pid = os.getpid() self._shows_names = {} self.alive = True self._lock = Lock() self.setup() logger.info("Starting master on pid %s" % self.pid)
def __init__(self, controller): self.controller = controller # init signals logger.info('Registering signals...') for sig in self.SIGNALS: signal.signal(sig, self.signal) # Don't let SIGQUIT and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGQUIT, False) signal.siginterrupt(signal.SIGUSR1, False)
def close_all(self): papa_sockets = 0 for sock in self.values(): sock.close() if isinstance(sock, PapaSocketProxy): papa_sockets += 1 if papa_sockets: with papa.Papa() as p: procs = p.list_processes('circus.*') if not procs: logger.info('removing all papa sockets') p.remove_sockets('circus.*') if p.exit_if_idle(): logger.info('closing papa')
def signal(self, sig, frame=None): signame = self.SIG_NAMES.get(sig) logger.info('Got signal SIG_%s' % signame.upper()) if signame is not None: try: handler = getattr(self, "handle_%s" % signame) handler() except AttributeError: pass except Exception as e: tb = traceback.format_exc() logger.error("error: %s [%s]" % (e, tb)) sys.exit(1)
def start(self): """Start. """ if not self.stopped: return self.stopped = False self.reap_processes() self.manage_processes() if self.stdout_redirector is not None: self.stdout_redirector.start() if self.stderr_redirector is not None: self.stderr_redirector.start() logger.info('%s started' % self.name) self.send_msg("start", {"time": time.time()})
def restart(self, inside_circusd=False, watcher_iter_func=None): if inside_circusd: self._restarting = True logger.info('Arbiter exiting') self._stopping = True yield self._stop_watchers(close_output_streams=True, for_shutdown=False) if self._provided_loop: cb = self.stop_controller_and_close_sockets self.loop.add_callback(cb) else: # stop_controller_and_close_sockets will be # called in the end of start() method self.loop.add_callback(self.loop.stop) else: yield self._stop_watchers(watcher_iter_func=watcher_iter_func) yield self._start_watchers(watcher_iter_func=watcher_iter_func)
def check(self, watcher_name): timeline = self.timelines[watcher_name] if watcher_name in self.configs: conf = self.configs[watcher_name] else: conf = self.update_conf(watcher_name) # if the watcher is not activated, we skip it if not to_bool(self._get_conf(conf, 'active')): # nothing to do here return tries = self.tries.get(watcher_name, 0) if len(timeline) == self._get_conf(conf, 'attempts'): duration = timeline[-1] - timeline[0] - self.check_delay if duration <= self._get_conf(conf, 'window'): max_retry = self._get_conf(conf, 'max_retry') if tries < max_retry or max_retry == INFINITE_RETRY: next_tries = tries + 1 logger.info( "%s: flapping detected: retry in %2ds " "(attempt number %s)", watcher_name, self._get_conf(conf, 'retry_in'), next_tries) self.cast("stop", name=watcher_name) self.timelines[watcher_name] = [] self.tries[watcher_name] = next_tries def _start(): self.cast("start", name=watcher_name) timer = Timer(self._get_conf(conf, 'retry_in'), _start) timer.start() self.timers[watcher_name] = timer else: logger.info( "%s: flapping detected: reached max retry limit", watcher_name) self.timelines[watcher_name] = [] self.tries[watcher_name] = 0 self.cast("stop", name=watcher_name) else: self.timelines[watcher_name] = [] self.tries[watcher_name] = 0