def look_after(self):
"""Checks for the watchdoged watchers and restart a process if no
received watchdog after the loop_rate * max_count period.
"""
# if first check, do a full discovery first.
if self.starting:
self._discover_monitored_pids()
self.starting = False
max_timeout = self.loop_rate * self.max_count
too_old_time = time.time() - max_timeout
for pid, detail in self.pid_status.items():
if detail['last_activity'] < too_old_time:
logger.info("watcher:%s, pid:%s is not responding. Kill it !",
detail['watcher'], pid)
props = dict(name=detail['watcher'], pid=int(pid))
if self.stop_signal is not None:
props['signum'] = self.stop_signal
if self.graceful_timeout is not None:
props['graceful_timeout'] = self.graceful_timeout
self.cast('kill', **props)
# Trusting watcher to eventually stop the process after
# graceful timeout
del self.pid_status[pid]
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
watcher_name, action, msg = self.split_data(data)
logger.debug("received data from circusd: watcher.%s.%s, %s",
watcher_name, action, msg)
# check if monitored watchers:
if self._match_watcher_name(watcher_name):
try:
message = self.load_message(msg)
except ValueError:
logger.error("Error while decoding json for message: %s", msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if action == "spawn":
self.pid_status[pid] = dict(watcher=watcher_name,
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
watcher_name, pid)
# very questionable fix for Py3 here!
# had to add check for pid in self.pid_status
elif action == "reap" and pid in self.pid_status:
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'], pid)
def start(self):
"""Start.
"""
if not self.stopped:
return
if self.on_demand and not self.arbiter.socket_event:
return
self.stopped = False
if not self.call_hook("before_start"):
logger.debug("Aborting startup")
self.stopped = True
return False
self._create_redirectors()
self.reap_processes()
self.spawn_processes()
if not self.call_hook("after_start"):
logger.debug("Aborting startup")
self.stop()
return False
if self.stdout_redirector is not None:
self.stdout_redirector.start()
if self.stderr_redirector is not None:
self.stderr_redirector.start()
logger.info("%s started" % self.name)
self.notify_event("start", {"time": time.time()})
return True
def start(self):
"""Starts all the watchers.
If the ioloop has been provided during __init__() call,
starts all watchers as a standard coroutine
If the ioloop hasn't been provided during __init__() call (default),
starts all watchers and the eventloop (and blocks here). In this mode
the method MUST NOT yield anything because it's called as a standard
method.
"""
logger.info("Starting master on pid %s", self.pid)
self.initialize()
# start controller
self.ctrl.start()
self._restarting = False
try:
# initialize processes
logger.debug('Initializing watchers')
if self._provided_loop:
yield self.start_watchers()
else:
# start_watchers will be called just after the start_io_loop()
self.loop.add_future(self.start_watchers(), lambda x: None)
logger.info('Arbiter now waiting for commands')
self._running = True
if not self._provided_loop:
# If an event loop is not provided, block at this line
self.start_io_loop()
finally:
if not self._provided_loop:
# If an event loop is not provided, do some cleaning
self.stop_controller_and_close_sockets()
raise gen.Return(self._restarting)
def _stop(self, close_output_streams=False, for_shutdown=False):
if self.is_stopped():
return
self._status = "stopping"
skip = for_shutdown and self.use_papa
if not skip:
logger.debug('stopping the %s watcher' % self.name)
logger.debug('gracefully stopping processes [%s] for %ss' %
(self.name, self.graceful_timeout))
# We ignore the hook result
self.call_hook('before_stop')
yield self.kill_processes()
self.reap_processes()
# stop redirectors
if self.stream_redirector:
self.stream_redirector.stop()
self.stream_redirector = None
if close_output_streams:
if self.stdout_stream and hasattr(self.stdout_stream, 'close'):
self.stdout_stream.close()
if self.stderr_stream and hasattr(self.stderr_stream, 'close'):
self.stderr_stream.close()
# notify about the stop
if skip:
logger.info('%s left running in papa', self.name)
else:
if self.evpub_socket is not None:
self.notify_event("stop", {"time": time.time()})
self._status = "stopped"
# We ignore the hook result
self.call_hook('after_stop')
logger.info('%s stopped', self.name)
def stop(self):
"""Stop.
"""
logger.debug('stopping the %s watcher' % self.name)
# stop redirectors
if self.stdout_redirector is not None:
self.stdout_redirector.stop()
self.stdout_redirector = None
if self.stderr_redirector is not None:
self.stderr_redirector.stop()
self.stderr_redirector = None
limit = time.time() + self.graceful_timeout
logger.debug('gracefully stopping processes [%s] for %ss' %
(self.name, self.graceful_timeout))
# We ignore the hook result
self.call_hook('before_stop')
while self.get_active_processes() and time.time() < limit:
self.kill_processes(signal.SIGTERM)
self.reap_processes()
self.kill_processes(signal.SIGKILL)
if self.evpub_socket is not None:
self.notify_event("stop", {"time": time.time()})
self.stopped = True
# We ignore the hook result
self.call_hook('after_stop')
logger.info('%s stopped', self.name)
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
watcher_name, action, msg = self.split_data(data)
logger.debug("received data from circusd: watcher.%s.%s, %s",
watcher_name, action, msg)
# check if monitored watchers:
if self._match_watcher_name(watcher_name):
try:
message = self.load_message(msg)
except ValueError:
logger.error("Error while decoding json for message: %s",
msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if action == "spawn":
self.pid_status[pid] = dict(watcher=watcher_name,
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
watcher_name,
pid)
# very questionable fix for Py3 here!
# had to add check for pid in self.pid_status
elif action == "reap" and pid in self.pid_status:
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'],
pid)
def start(self):
"""Starts all the watchers.
The start command is an infinite loop that waits
for any command from a client and that watches all the
processes and restarts them if needed.
"""
logger.info("Starting master on pid %s", self.pid)
self.initialize()
# start controller
self.ctrl.start()
# initialize processes
logger.debug("Initializing watchers")
for watcher in self.iter_watchers():
watcher.start()
logger.info("Arbiter now waiting for commands")
while True:
try:
self.loop.start()
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise
else:
break
self.ctrl.stop()
self.evpub_socket.close()
def _stop(self, close_output_streams=False, for_shutdown=False):
if self.is_stopped():
return
self._status = "stopping"
skip = for_shutdown and self.use_papa
if not skip:
logger.debug('stopping the %s watcher' % self.name)
logger.debug('gracefully stopping processes [%s] for %ss' % (
self.name, self.graceful_timeout))
# We ignore the hook result
self.call_hook('before_stop')
yield self.kill_processes()
self.reap_processes()
# stop redirectors
if self.stream_redirector:
self.stream_redirector.stop()
self.stream_redirector = None
if close_output_streams:
if self.stdout_stream and hasattr(self.stdout_stream, 'close'):
self.stdout_stream.close()
if self.stderr_stream and hasattr(self.stderr_stream, 'close'):
self.stderr_stream.close()
# notify about the stop
if skip:
logger.info('%s left running in papa', self.name)
else:
if self.evpub_socket is not None:
self.notify_event("stop", {"time": time.time()})
self._status = "stopped"
# We ignore the hook result
self.call_hook('after_stop')
logger.info('%s stopped', self.name)
def stop(self):
"""Stop.
"""
logger.debug('stopping the %s watcher' % self.name)
# stop redirectors
if self.stdout_redirector is not None:
self.stdout_redirector.kill()
if self.stderr_redirector is not None:
self.stderr_redirector.kill()
limit = time.time() + self.graceful_timeout
logger.debug('gracefully stopping processes [%s] for %ss' % (
self.name, self.graceful_timeout))
while self.get_active_processes() and time.time() < limit:
self.kill_processes(signal.SIGTERM)
try:
time.sleep(0.1)
except KeyboardInterrupt:
pass
self.reap_processes()
self.kill_processes(signal.SIGKILL)
if self.evpub_socket is not None:
self.notify_event("stop", {"time": time.time()})
self.stopped = True
logger.info('%s stopped', self.name)
def restart(self):
"""Restart.
"""
self.notify_event("restart", {"time": time.time()})
self.stop()
self.start()
logger.info('%s restarted', self.name)
def spawn_fly(self):
if self.stopped:
return
self._fly_counter += 1
nb_tries = 0
while nb_tries < self.max_retry:
fly = None
try:
fly = Fly(self._fly_counter,
self.cmd,
wdir=self.working_dir,
shell=self.shell,
uid=self.uid,
gid=self.gid,
env=self.env)
self.flies[self._fly_counter] = fly
logger.info('running %s fly [pid %d]' % (self.name, fly.pid))
except OSError, e:
logger.warning('error in %r: %s' % (self.name, str(e)))
if fly is None:
nb_tries += 1
continue
else:
return
def restart(self):
""" restart
"""
self.send_msg("restart", {"time": time.time()})
self.stop()
self.start()
logger.info('%s restarted' % self.name)
def restart(self):
"""Restart.
"""
self.send_msg("restart", {"time": time.time()})
self.stop()
self.start()
logger.info("%s restarted", self.name)
def restart(self):
"""Restart.
"""
self.notify_event("restart", {"time": time.time()})
self.stop()
self.start()
logger.info('%s restarted', self.name)
def stop(self):
"""Stop.
"""
logger.debug('stopping the %s watcher' % self.name)
# stop redirectors
if self.stdout_redirector is not None:
self.stdout_redirector.kill()
if self.stderr_redirector is not None:
self.stderr_redirector.kill()
limit = time.time() + self.graceful_timeout
logger.debug('gracefully stopping processes [%s] for %ss' % (
self.name, self.graceful_timeout))
# We ignore the hook result
self.call_hook('before_stop')
while self.get_active_processes() and time.time() < limit:
self.kill_processes(signal.SIGTERM)
self.reap_processes()
self.kill_processes(signal.SIGKILL)
if self.evpub_socket is not None:
self.notify_event("stop", {"time": time.time()})
self.stopped = True
# We ignore the hook result
self.call_hook('after_stop')
logger.info('%s stopped', self.name)
def start(self):
"""Start.
"""
if not self.stopped:
return
self.stopped = False
if not self.call_hook('before_start'):
logger.debug('Aborting startup')
self.stopped = True
return False
self._create_redirectors()
self.reap_processes()
self.spawn_processes()
if not self.call_hook('after_start'):
logger.debug('Aborting startup')
self.stop()
return False
if self.stdout_redirector is not None:
self.stdout_redirector.start()
if self.stderr_redirector is not None:
self.stderr_redirector.start()
logger.info('%s started' % self.name)
self.notify_event("start", {"time": time.time()})
return True
def _reload(self, graceful=True, sequential=False):
""" reload
"""
if not graceful and sequential:
logger.warn("with graceful=False, sequential=True is ignored")
if self.prereload_fn is not None:
self.prereload_fn(self)
if not graceful:
yield self._restart()
return
if self.is_stopped():
yield self._start()
elif self.send_hup:
for process in self.processes.values():
logger.info("SENDING HUP to %s" % process.pid)
process.send_signal(signal.SIGHUP)
else:
if sequential:
active_processes = self.get_active_processes()
for process in active_processes:
yield self.kill_process(process)
self.reap_process(process.pid)
self.spawn_process()
yield tornado_sleep(self.warmup_delay)
else:
for i in range(self.numprocesses):
self.spawn_process()
yield self.manage_processes()
self.notify_event("reload", {"time": time.time()})
logger.info('%s reloaded', self.name)
def start(self):
"""Starts all the watchers.
If the ioloop has been provided during __init__() call,
starts all watchers as a standard coroutine
If the ioloop hasn't been provided during __init__() call (default),
starts all watchers and the eventloop (and blocks here). In this mode
the method MUST NOT yield anything because it's called as a standard
method.
"""
logger.info("Starting master on pid %s", self.pid)
self.initialize()
# start controller
self.ctrl.start()
self._restarting = False
try:
# initialize processes
logger.debug('Initializing watchers')
if self._provided_loop:
yield self.start_watchers()
else:
# start_watchers will be called just after the start_io_loop()
self.loop.add_future(self.start_watchers(), lambda x: None)
logger.info('Arbiter now waiting for commands')
if not self._provided_loop:
# If an event loop is not provided, block at this line
self.start_io_loop()
finally:
if not self._provided_loop:
# If an event loop is not provided, do some cleaning
self.stop_controller_and_close_sockets()
raise gen.Return(self._restarting)
def stop(self):
"""Stop.
"""
logger.debug("stopping the %s watcher" % self.name)
# stop redirectors
if self.stdout_redirector is not None:
self.stdout_redirector.kill()
if self.stderr_redirector is not None:
self.stderr_redirector.kill()
limit = time.time() + self.graceful_timeout
logger.debug("gracefully stopping processes [%s] for %ss" % (self.name, self.graceful_timeout))
while self.get_active_pids() and time.time() < limit:
self.kill_processes(signal.SIGTERM)
time.sleep(0.1)
self.reap_processes()
self.kill_processes(signal.SIGKILL)
if self.evpub_socket is not None:
self.send_msg("stop", {"time": time.time()})
self.stopped = True
logger.info("%s stopped", self.name)
def spawn_fly(self):
""" spawn fly
"""
if self.stopped:
return
self._fly_counter += 1
nb_tries = 0
while nb_tries < self.max_retry:
fly = None
try:
fly = Fly(self._fly_counter, self.cmd,
working_dir=self.working_dir, shell=self.shell,
uid=self.uid, gid=self.gid, env=self.env)
self.flies[self._fly_counter] = fly
logger.info('running %s fly [pid %d]' % (self.name, fly.pid))
except OSError, e:
logger.warning('error in %r: %s' % (self.name, str(e)))
if fly is None:
nb_tries += 1
continue
else:
self.send_msg("spawn", {"fly_id": fly.wid,
"fly_pid": fly.pid,
"time": time.time()})
return
def look_after(self):
# Get concerned watchers
# Note: watchers and their configuration can change at any time
watchers = self.call('list')['watchers']
if self.watchers != '*':
watchers = [w for w in watchers if w in self.watchers.split(',')]
for watcher in list(self.infos.keys()):
if watcher not in watchers:
del self.infos[watcher] # Clean obsolete watchers
# Check modifications
for w in watchers:
# We make sure not to reload to often (wait at least 2secs)
if self.infos[w].get('last_restart') < time.time() - 2:
w_info = self.call('get', name=w, keys=['cmd', 'working_dir'])
previous_path = self.infos[w].get('path')
if self.use_working_dir:
current_path = w_info['options']['working_dir']
else:
current_path = w_info['options']['cmd']
self.infos[w]['path'] = current_path
self.infos[w]['mtime'] = self.mtime_of_path(current_path)
if self.is_modified(w, previous_path):
if self.use_reload:
logger.info('%s modified. Reloading.',
self.infos[w]['path'])
self.call('reload', name=w)
else:
logger.info('%s modified. Reloading.',
self.infos[w]['path'])
self.call('restart', name=w)
self.infos[w]['last_restart'] = time.time()
def start(self):
logger.info('Starting the stats streamer')
self._init()
logger.debug('Initial list is ' + str(self._pids))
self.running = True
self.stats.start()
self.publisher.start()
logger.debug('Now looping to get circusd events')
while self.running:
try:
self.loop.start()
except zmq.ZMQError as e:
logger.debug(str(e))
if e.errno == errno.EINTR:
continue
elif e.errno == zmq.ETERM:
break
else:
logger.debug("got an unexpected error %s (%s)", str(e),
e.errno)
raise
else:
break
self.sub_socket.close()
def look_after(self):
"""Checks for the watchdoged watchers and restart a process if no
received watchdog after the loop_rate * max_count period.
"""
# if first check, do a full discovery first.
if self.starting:
self._discover_monitored_pids()
self.starting = False
max_timeout = self.loop_rate * self.max_count
too_old_time = time.time() - max_timeout
for pid, detail in self.pid_status.items():
if detail["last_activity"] < too_old_time:
logger.info("watcher:%s, pid:%s is not responding. Kill it !", detail["watcher"], pid)
props = dict(name=detail["watcher"], pid=int(pid))
if self.stop_signal is not None:
props["signum"] = self.stop_signal
if self.graceful_timeout is not None:
props["graceful_timeout"] = self.graceful_timeout
self.cast("kill", **props)
# Trusting watcher to eventually stop the process after
# graceful timeout
del self.pid_status[pid]
def stop(self):
"""Stop.
"""
logger.debug('stopping the %s watcher' % self.name)
# stop redirectors
if self.stdout_redirector is not None:
self.stdout_redirector.kill()
if self.stderr_redirector is not None:
self.stderr_redirector.kill()
limit = time.time() + self.graceful_timeout
logger.debug('gracefully stopping processes [%s] for %ss' %
(self.name, self.graceful_timeout))
while self.get_active_processes() and time.time() < limit:
self.kill_processes(signal.SIGTERM)
try:
time.sleep(0.1)
except KeyboardInterrupt:
pass
self.reap_processes()
self.kill_processes(signal.SIGKILL)
if self.evpub_socket is not None:
self.notify_event("stop", {"time": time.time()})
self.stopped = True
logger.info('%s stopped', self.name)
def start(self):
"""Starts all the watchers.
The start command is an infinite loop that waits
for any command from a client and that watches all the
processes and restarts them if needed.
"""
logger.info("Starting master on pid %s", self.pid)
self.initialize()
# start controller
self.ctrl.start()
# initialize processes
logger.debug('Initializing watchers')
for watcher in self.iter_watchers():
watcher.start()
logger.info('Arbiter now waiting for commands')
while True:
try:
self.loop.start()
except zmq.ZMQError as e:
if e.errno == errno.EINTR:
continue
else:
raise
else:
break
self.ctrl.stop()
self.evpub_socket.close()
def __init__(self, controller):
self.controller = controller
# init signals
logger.info('Registering signals...')
self._old = {}
self._register()
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
topic, msg = data
topic_parts = topic.split(".")
logger.debug("received data from circusd: %s, %s", topic_parts, msg)
# check if monitored watchers:
if (topic_parts[0] == 'watcher' and
self._match_watcher_name(topic_parts[1])):
try:
message = json.loads(msg)
except ValueError:
logger.error("Error while decoding json for message: %s",
msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if topic_parts[2] == "spawn":
self.pid_status[pid] = dict(watcher=topic_parts[1],
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
topic_parts[1],
pid)
elif topic_parts[2] == "reap":
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'],
pid)
def restart(self):
"""Restart.
"""
self.send_msg("restart", {"time": time.time()})
self.stop()
self.start()
logger.info('%s restarted', self.name)
def handle_recv(self, data):
"""Handle received message from circusd
We need to handle two messages:
- spawn: add a new monitored child pid
- reap: remove a killed child pid from monitoring
"""
topic, msg = data
topic_parts = topic.split(".")
logger.debug("received data from circusd: %s, %s", topic_parts, msg)
# check if monitored watchers:
if (topic_parts[0] == 'watcher'
and self._match_watcher_name(topic_parts[1])):
try:
message = json.loads(msg)
except ValueError:
logger.error("Error while decoding json for message: %s", msg)
else:
if "process_pid" not in message:
logger.warning('no process_pid in message')
return
pid = str(message.get("process_pid"))
if topic_parts[2] == "spawn":
self.pid_status[pid] = dict(watcher=topic_parts[1],
last_activity=time.time())
logger.info("added new monitored pid for %s:%s",
topic_parts[1], pid)
elif topic_parts[2] == "reap":
old_pid = self.pid_status.pop(pid)
logger.info("removed monitored pid for %s:%s",
old_pid['watcher'], pid)
def _reload(self, graceful=True, sequential=False):
""" reload
"""
if not graceful and sequential:
logger.warn("with graceful=False, sequential=True is ignored")
if self.prereload_fn is not None:
self.prereload_fn(self)
if not graceful:
yield self._restart()
return
if self.is_stopped():
yield self._start()
elif self.send_hup:
for process in self.processes.values():
logger.info("SENDING HUP to %s" % process.pid)
process.send_signal(signal.SIGHUP)
else:
if sequential:
active_processes = self.get_active_processes()
for process in active_processes:
yield self.kill_process(process)
self.reap_process(process.pid)
self.spawn_process()
yield tornado_sleep(self.warmup_delay)
else:
for i in range(self.numprocesses):
self.spawn_process()
yield self.manage_processes()
self.notify_event("reload", {"time": time.time()})
logger.info('%s reloaded', self.name)
def _start(self):
"""Start.
"""
if not self.is_stopped():
return
if self.on_demand and not self.arbiter.socket_event:
return
if not self.call_hook('before_start'):
logger.debug('Aborting startup')
return
self._status = "starting"
self._create_redirectors()
self.reap_processes()
yield self.spawn_processes()
if not self.call_hook('after_start'):
logger.debug('Aborting startup')
yield self._stop()
return
if self.stdout_redirector is not None:
self.stdout_redirector.start()
if self.stderr_redirector is not None:
self.stderr_redirector.start()
self._status = "active"
logger.info('%s started' % self.name)
self.notify_event("start", {"time": time.time()})
def start(self):
"""Start.
"""
if not self.stopped:
return
if self.on_demand and not self.arbiter.socket_event:
return
self.stopped = False
if not self.call_hook('before_start'):
logger.debug('Aborting startup')
self.stopped = True
return False
self._create_redirectors()
self.reap_processes()
self.spawn_processes()
if not self.call_hook('after_start'):
logger.debug('Aborting startup')
self.stop()
return False
if self.stdout_redirector is not None:
self.stdout_redirector.start()
if self.stderr_redirector is not None:
self.stderr_redirector.start()
logger.info('%s started' % self.name)
self.notify_event("start", {"time": time.time()})
return True
def check(self, watcher_name):
timeline = self.timelines[watcher_name]
if watcher_name in self.configs:
conf = self.configs[watcher_name]
else:
conf = self.update_conf(watcher_name)
tries = self.tries.get(watcher_name, 0)
if len(timeline) == conf["times"]:
duration = timeline[-1] - timeline[0] - self.check_delay
if duration <= conf["within"]:
if tries < conf["max_retry"]:
logger.info("%s: flapping detected: retry in %2ds", watcher_name, conf["retry_in"])
self.cast(make_message("stop", name=watcher_name))
self.timelines[watcher_name] = []
self.tries[watcher_name] = tries + 1
def _start():
self.cast(make_message("start", name=watcher_name))
timer = Timer(conf["retry_in"], _start)
timer.start()
self.timers[watcher_name] = timer
else:
logger.info("%s: flapping detected: max retry limit", watcher_name)
self.timelines[watcher_name] = []
self.tries[watcher_name] = 0
self.cast(make_message("stop", name=watcher_name))
else:
self.timelines[watcher_name] = []
self.tries[watcher_name] = 0
def main():
parser = argparse.ArgumentParser(description="Runs a plugin.")
parser.add_argument("--endpoint", help="The circusd ZeroMQ socket to connect to", default=DEFAULT_ENDPOINT_DEALER)
parser.add_argument(
"--pubsub", help="The circusd ZeroMQ pub/sub socket to connect to", default=DEFAULT_ENDPOINT_SUB
)
parser.add_argument("--config", help="The plugin configuration", default=None)
parser.add_argument("--version", action="store_true", default=False, help="Displays Circus version and exits.")
parser.add_argument("--check-delay", type=float, default=5.0, help="Checck delay.")
parser.add_argument("plugin", help="Fully qualified name of the plugin class.", nargs="?")
parser.add_argument("--log-level", dest="loglevel", default="info", help="log level")
parser.add_argument("--log-output", dest="logoutput", default="-", help="log output")
parser.add_argument("--ssh", default=None, help="SSH Server")
args = parser.parse_args()
if args.version:
print(__version__)
sys.exit(0)
if args.plugin is None:
parser.print_usage()
sys.exit(0)
# configure the logger
loglevel = LOG_LEVELS.get(args.loglevel.lower(), logging.INFO)
logger.setLevel(loglevel)
if args.logoutput == "-":
h = logging.StreamHandler()
else:
h = logging.handlers.WatchedFileHandler(args.logoutput)
close_on_exec(h.stream.fileno())
fmt = logging.Formatter(LOG_FMT, LOG_DATE_FMT)
h.setFormatter(fmt)
logger.addHandler(h)
# load the plugin and run it.
logger.info("Loading the plugin...")
logger.info("Endpoint: %r" % args.endpoint)
logger.info("Pub/sub: %r" % args.pubsub)
plugin = resolve_name(args.plugin)(args.endpoint, args.pubsub, args.check_delay, args.ssh, **_str2cfg(args.config))
logger.info("Starting")
try:
plugin.start()
except KeyboardInterrupt:
pass
finally:
logger.info("Stopping")
plugin.stop()
sys.exit(0)
def start(self):
if not self.stopped:
return
self.stopped = False
self.reap_flies()
self.manage_flies()
logger.info('%s started' % self.name)
def start(self):
if not self.stopped:
return
self.stopped = False
self.reap_flies()
self.manage_flies()
logger.info('%s started' % self.name)
def _stop(self):
logger.info('Arbiter exiting')
self._stopping = True
yield self._stop_watchers(close_output_streams=True)
if self._provided_loop:
cb = self.stop_controller_and_close_sockets
self.loop.add_callback(cb)
else:
self.loop.add_timeout(time.time() + 1, self._stop_cb)
def restart(self):
"""Restart.
"""
self.notify_event("restart", {"time": time.time()})
self.stop()
if self.start():
logger.info('%s restarted', self.name)
else:
logger.info('Failed to restart %s', self.name)
def _stop(self):
logger.info('Arbiter exiting')
self._stopping = True
yield self._stop_watchers(close_output_streams=True)
if self._provided_loop:
cb = self.stop_controller_and_close_sockets
self.loop.add_callback(cb)
else:
self.loop.add_timeout(time.time() + 1, self._stop_cb)
def restart(self):
"""Restart.
"""
self.notify_event("restart", {"time": time.time()})
self.stop()
if self.start():
logger.info('%s restarted', self.name)
else:
logger.info('Failed to restart %s', self.name)
def stop_watchers(self, graceful=True, stop_alive=False):
if stop_alive:
logger.info("Arbiter exiting")
if not self.alive:
return
self.alive = False
for watcher in self.watchers:
watcher.stop(graceful=graceful)
def _ensure_ioloop(self):
if self.loop is None:
self.loop = ioloop.IOLoop.current()
def handle_callback_exception(callback):
logger.error("Exception in callback %r", callback, exc_info=True)
if not hasattr(self.loop, 'handle_callback_exception'):
logger.info('Installing handle_callback_exception to loop')
setattr(self.loop, 'handle_callback_exception', handle_callback_exception)
def handle_reload(self, *args):
if self.send_hup:
for wid, fly in self.flies.items():
logger.info("SEND HUP to %s [%s]" % (wid, fly.pid))
fly.send_signal(signal.SIGHUP)
else:
for i in range(self.num_flies):
self.spawn_fly()
self.manage_flies()
return "ok"
def handle_reload(self, *args):
if self.send_hup:
for wid, fly in self.flies.items():
logger.info("SEND HUP to %s [%s]" % (wid, fly.pid))
fly.send_signal(signal.SIGHUP)
else:
for i in range(self.num_flies):
self.spawn_fly()
self.manage_flies()
return "ok"
def stop_watchers(self, stop_alive=False):
if not self.alive:
return
if stop_alive:
logger.info('Arbiter exiting')
self.alive = False
for watcher in self.iter_watchers():
watcher.stop()
def stop_watchers(self, stop_alive=False):
if not self.alive:
return
if stop_alive:
logger.info('Arbiter exiting')
self.alive = False
for watcher in self.iter_watchers():
watcher.stop()
def stop(self):
# stop all the periodic callbacks running
for callback in self._callbacks.values():
callback.stop()
self.loop.stop()
self.ctx.destroy(0)
self.publisher.stop()
self.stopped = True
self.running = False
logger.info('Stats streamer stopped')
def stop(self):
# stop all the periodic callbacks running
for callback in self._callbacks.values():
callback.stop()
self.loop.stop()
self.ctx.destroy(0)
self.publisher.stop()
self.stopped = True
self.running = False
logger.info('Stats streamer stopped')
def start(self):
"""Start.
"""
if not self.stopped:
return
self.stopped = False
self.reap_processes()
self.manage_processes()
logger.info('%s started' % self.name)
self.send_msg("start", {"time": time.time()})
def __stop(self, for_shutdown=False):
logger.info('Arbiter exiting')
self._stopping = True
yield self._stop_watchers(close_output_streams=True,
for_shutdown=for_shutdown)
if self._provided_loop:
cb = self.stop_controller_and_close_sockets
self.loop.add_callback(cb)
else:
# stop_controller_and_close_sockets will be
# called in the end of start() method
self.loop.add_callback(self.loop.stop)
def __init__(self, shows, endpoint, check_delay=1., ipc_path=None):
self.shows = shows
self.endpoint = endpoint
self.check_delay = check_delay
self.ipc_path = ipc_path
self.ctrl = Controller(endpoint, self, self.check_delay, self.ipc_path)
self.pid = os.getpid()
self._shows_names = {}
self.alive = True
self._lock = Lock()
self.setup()
logger.info("Starting master on pid %s" % self.pid)
def __init__(self, controller):
self.controller = controller
# init signals
logger.info('Registering signals...')
for sig in self.SIGNALS:
signal.signal(sig, self.signal)
# Don't let SIGQUIT and SIGUSR1 disturb active requests
# by interrupting system calls
if hasattr(signal, 'siginterrupt'): # python >= 2.6
signal.siginterrupt(signal.SIGQUIT, False)
signal.siginterrupt(signal.SIGUSR1, False)
def close_all(self):
papa_sockets = 0
for sock in self.values():
sock.close()
if isinstance(sock, PapaSocketProxy):
papa_sockets += 1
if papa_sockets:
with papa.Papa() as p:
procs = p.list_processes('circus.*')
if not procs:
logger.info('removing all papa sockets')
p.remove_sockets('circus.*')
if p.exit_if_idle():
logger.info('closing papa')
def signal(self, sig, frame=None):
signame = self.SIG_NAMES.get(sig)
logger.info('Got signal SIG_%s' % signame.upper())
if signame is not None:
try:
handler = getattr(self, "handle_%s" % signame)
handler()
except AttributeError:
pass
except Exception as e:
tb = traceback.format_exc()
logger.error("error: %s [%s]" % (e, tb))
sys.exit(1)
def start(self):
"""Start.
"""
if not self.stopped:
return
self.stopped = False
self.reap_processes()
self.manage_processes()
if self.stdout_redirector is not None:
self.stdout_redirector.start()
if self.stderr_redirector is not None:
self.stderr_redirector.start()
logger.info('%s started' % self.name)
self.send_msg("start", {"time": time.time()})
def restart(self, inside_circusd=False, watcher_iter_func=None):
if inside_circusd:
self._restarting = True
logger.info('Arbiter exiting')
self._stopping = True
yield self._stop_watchers(close_output_streams=True,
for_shutdown=False)
if self._provided_loop:
cb = self.stop_controller_and_close_sockets
self.loop.add_callback(cb)
else:
# stop_controller_and_close_sockets will be
# called in the end of start() method
self.loop.add_callback(self.loop.stop)
else:
yield self._stop_watchers(watcher_iter_func=watcher_iter_func)
yield self._start_watchers(watcher_iter_func=watcher_iter_func)
def check(self, watcher_name):
timeline = self.timelines[watcher_name]
if watcher_name in self.configs:
conf = self.configs[watcher_name]
else:
conf = self.update_conf(watcher_name)
# if the watcher is not activated, we skip it
if not to_bool(self._get_conf(conf, 'active')):
# nothing to do here
return
tries = self.tries.get(watcher_name, 0)
if len(timeline) == self._get_conf(conf, 'attempts'):
duration = timeline[-1] - timeline[0] - self.check_delay
if duration <= self._get_conf(conf, 'window'):
max_retry = self._get_conf(conf, 'max_retry')
if tries < max_retry or max_retry == INFINITE_RETRY:
next_tries = tries + 1
logger.info(
"%s: flapping detected: retry in %2ds "
"(attempt number %s)", watcher_name,
self._get_conf(conf, 'retry_in'), next_tries)
self.cast("stop", name=watcher_name)
self.timelines[watcher_name] = []
self.tries[watcher_name] = next_tries
def _start():
self.cast("start", name=watcher_name)
timer = Timer(self._get_conf(conf, 'retry_in'), _start)
timer.start()
self.timers[watcher_name] = timer
else:
logger.info(
"%s: flapping detected: reached max retry limit",
watcher_name)
self.timelines[watcher_name] = []
self.tries[watcher_name] = 0
self.cast("stop", name=watcher_name)
else:
self.timelines[watcher_name] = []
self.tries[watcher_name] = 0
