def validate(self, props):
super(Signal, self).validate(props)
if 'childpid' in props and not 'pid' in props:
raise ArgumentError('cannot specify childpid without pid')
try:
props['signum'] = to_signum(props['signum'])
except ValueError:
raise MessageError('signal invalid')
def validate(self, props):
super(Kill, self).validate(props)
if 'pid' in props:
props['pid'] = int(props['pid'])
try:
if 'signum' in props:
props['signum'] = to_signum(props['signum'])
except ValueError:
raise MessageError('signal invalid')
def validate(self, props):
super(Signal, self).validate(props)
if "childpid" in props and not "pid" in props:
raise ArgumentError("cannot specify childpid without pid")
try:
props["signum"] = to_signum(props["signum"])
except ValueError:
raise MessageError("signal invalid")
props["pid"] = int(props["pid"])
def __init__(self, endpoint, pubsub_endpoint, check_delay, ssh_server, **config):
super(WatchDog, self).__init__(endpoint, pubsub_endpoint, check_delay, ssh_server=ssh_server)
self.loop_rate = float(config.get("loop_rate", 60)) # in seconds
self.watchers_regex = config.get("watchers_regex", ".*")
self.msg_regex = config.get("msg_regex", "^(?P<pid>.*);(?P<timestamp>.*)$")
self.max_count = int(config.get("max_count", 3))
self.watchdog_ip = config.get("ip", "127.0.0.1")
self.watchdog_port = int(config.get("port", 1664))
self.stop_signal = config.get("watchers_stop_signal")
if self.stop_signal:
self.stop_signal = util.to_signum(self.stop_signal)
self.graceful_timeout = config.get("watchers_graceful_timeout")
if self.graceful_timeout:
self.graceful_timeout = float(self.graceful_timeout)
self.pid_status = dict()
self.period = None
self.starting = True
def __init__(self, endpoint, pubsub_endpoint, check_delay, ssh_server,
**config):
super(WatchDog, self).__init__(endpoint,
pubsub_endpoint,
check_delay,
ssh_server=ssh_server)
self.loop_rate = float(config.get("loop_rate", 60)) # in seconds
self.watchers_regex = config.get("watchers_regex", ".*")
self.msg_regex = config.get("msg_regex",
"^(?P<pid>.*);(?P<timestamp>.*)$")
self.max_count = int(config.get("max_count", 3))
self.watchdog_ip = config.get("ip", "127.0.0.1")
self.watchdog_port = int(config.get("port", 1664))
self.stop_signal = config.get("watchers_stop_signal")
if self.stop_signal:
self.stop_signal = to_signum(self.stop_signal)
self.graceful_timeout = config.get("watchers_graceful_timeout")
if self.graceful_timeout:
self.graceful_timeout = float(self.graceful_timeout)
self.pid_status = dict()
self.period = None
self.starting = True
def set_opt(self, key, val):
"""Set a watcher option.
This function set the watcher options. unknown keys are ignored.
This function return an action number:
- 0: trigger the process management
- 1: trigger a graceful reload of the processes;
"""
action = 0
if key in self._options:
self._options[key] = val
action = -1 # XXX for now does not trigger a reload
elif key == "numprocesses":
val = int(val)
if val < 0:
val = 0
if self.singleton and val > 1:
raise ValueError('Singleton watcher has a single process')
self.numprocesses = val
elif key == "warmup_delay":
self.warmup_delay = float(val)
elif key == "working_dir":
self.working_dir = val
action = 1
elif key == "uid":
self.uid = util.to_uid(val)
action = 1
elif key == "gid":
self.gid = util.to_gid(val)
action = 1
elif key == "send_hup":
self.send_hup = val
elif key == "stop_signal":
self.stop_signal = util.to_signum(val)
elif key == "stop_children":
self.stop_children = util.to_bool(val)
elif key == "shell":
self.shell = val
action = 1
elif key == "env":
if PY2 and IS_WINDOWS:
# Windows on Python 2 does not accept Unicode values
# in env dictionary
self.env = dict((b(k), b(v)) for k, v in val.iteritems())
else:
self.env = val
action = 1
elif key == "cmd":
self.cmd = val
action = 1
elif key == "args":
self.args = val
action = 1
elif key == "graceful_timeout":
self.graceful_timeout = float(val)
action = -1
elif key == "max_age":
self.max_age = int(val)
action = 1
elif key == "max_age_variance":
self.max_age_variance = int(val)
action = 1
elif (key.startswith('stdout_stream')
or key.startswith('stderr_stream')):
action = self._reload_stream(key, val)
elif key.startswith('hooks'):
val = val.split(',')
if len(val) == 2:
ignore_error = util.to_bool(val[1])
else:
ignore_error = False
hook = val[0]
self._reload_hook(key, hook, ignore_error)
action = 0
# send update event
self.notify_event("updated", {"time": time.time()})
return action
def get_config(config_file):
if not os.path.exists(config_file):
raise IOError("the configuration file %r does not exist\n" %
config_file)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# reading the global environ first
def _upper(items):
return [(key.upper(), value) for key, value in items]
global_env = dict(_upper(os.environ.items()))
local_env = dict()
# update environments with [env] section
if 'env' in cfg.sections():
local_env.update(dict(_upper(cfg.items('env'))))
global_env.update(local_env)
# always set the cfg environment
cfg.set_env(global_env)
# main circus options
config['check_delay'] = dget('circus', 'check_delay', 5., float)
config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
DEFAULT_ENDPOINT_SUB)
config['multicast_endpoint'] = dget('circus', 'multicast_endpoint',
DEFAULT_ENDPOINT_MULTICAST)
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None)
config['statsd'] = dget('circus', 'statsd', False, bool)
config['umask'] = dget('circus', 'umask', None)
if config['umask']:
config['umask'] = int(config['umask'], 8)
if config['stats_endpoint'] is None:
config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS
elif not config['statsd']:
warnings.warn("You defined a stats_endpoint without "
"setting up statsd to True.",
DeprecationWarning)
config['statsd'] = True
config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
config['httpd'] = dget('circus', 'httpd', False, bool)
config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
config['debug'] = dget('circus', 'debug', False, bool)
config['pidfile'] = dget('circus', 'pidfile')
config['loglevel'] = dget('circus', 'loglevel')
config['logoutput'] = dget('circus', 'logoutput')
config['fqdn_prefix'] = dget('circus', 'fqdn_prefix', None, str)
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1].lower()
sock['so_reuseport'] = dget(section, "so_reuseport", False, bool)
sock['replace'] = dget(section, "replace", False, bool)
sockets.append(sock)
if section.startswith("plugin:"):
plugin = dict(cfg.items(section))
plugin['name'] = section
if 'priority' in plugin:
plugin['priority'] = int(plugin['priority'])
plugins.append(plugin)
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section, noreplace=True):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'stop_signal':
watcher['stop_signal'] = to_signum(val)
elif opt == 'stop_children':
watcher['stop_children'] = dget(section, 'stop_children',
False, bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timeout':
watcher['graceful_timeout'] = dget(
section, "graceful_timeout", 30, int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
elif opt == 'singleton':
watcher['singleton'] = dget(section, "singleton", False,
bool)
elif opt == 'copy_env':
watcher['copy_env'] = dget(section, "copy_env", False,
bool)
elif opt == 'copy_path':
watcher['copy_path'] = dget(section, "copy_path", False,
bool)
elif opt.startswith('hooks.'):
hook_name = opt[len('hooks.'):]
val = [elmt.strip() for elmt in val.split(',', 1)]
if len(val) == 1:
val.append(False)
else:
val[1] = to_boolean(val[1])
watcher['hooks'][hook_name] = val
elif opt == 'respawn':
watcher['respawn'] = dget(section, "respawn", True, bool)
elif opt == 'autostart':
watcher['autostart'] = dget(section, "autostart", True,
bool)
elif opt == 'close_child_stdout':
watcher['close_child_stdout'] = dget(section,
"close_child_stdout",
False, bool)
elif opt == 'close_child_stderr':
watcher['close_child_stderr'] = dget(section,
"close_child_stderr",
False, bool)
else:
# freeform
watcher[opt] = val
if watcher['copy_env']:
watcher['env'] = dict(global_env)
else:
watcher['env'] = dict(local_env)
watchers.append(watcher)
# Second pass to make sure env sections apply to all watchers.
def _extend(target, source):
for name, value in source:
if name in target:
continue
target[name] = value
def _expand_vars(target, key, env):
if isinstance(target[key], str):
target[key] = replace_gnu_args(target[key], env=env)
elif isinstance(target[key], dict):
for k in target[key].keys():
_expand_vars(target[key], k, env)
def _expand_section(section, env, exclude=None):
if exclude is None:
exclude = ('name', 'env')
for option in section.keys():
if option in exclude:
continue
_expand_vars(section, option, env)
# build environment for watcher sections
for section in cfg.sections():
if section.startswith('env:'):
section_elements = section.split("env:", 1)[1]
watcher_patterns = [s.strip() for s in section_elements.split(',')]
env_items = dict(_upper(cfg.items(section, noreplace=True)))
for pattern in watcher_patterns:
match = [w for w in watchers if fnmatch(w['name'], pattern)]
for watcher in match:
watcher['env'].update(env_items)
# expand environment for watcher sections
for watcher in watchers:
env = dict(global_env)
env.update(watcher['env'])
_expand_section(watcher, env)
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
def set_opt(self, key, val):
"""Set a watcher option.
This function set the watcher options. unknown keys are ignored.
This function return an action number:
- 0: trigger the process management
- 1: trigger a graceful reload of the processes;
"""
action = 0
if key in self._options:
self._options[key] = val
action = -1 # XXX for now does not trigger a reload
elif key == "numprocesses":
val = int(val)
if val < 0:
val = 0
if self.singleton and val > 1:
raise ValueError('Singleton watcher has a single process')
self.numprocesses = val
elif key == "warmup_delay":
self.warmup_delay = float(val)
elif key == "working_dir":
self.working_dir = val
action = 1
elif key == "uid":
self.uid = util.to_uid(val)
action = 1
elif key == "gid":
self.gid = util.to_gid(val)
action = 1
elif key == "send_hup":
self.send_hup = val
elif key == "stop_signal":
self.stop_signal = util.to_signum(val)
elif key == "stop_children":
self.stop_children = util.to_bool(val)
elif key == "shell":
self.shell = val
action = 1
elif key == "env":
if PY2 and IS_WINDOWS:
# Windows on Python 2 does not accept Unicode values
# in env dictionary
self.env = dict((b(k), b(v)) for k, v in val.iteritems())
else:
self.env = val
action = 1
elif key == "cmd":
self.cmd = val
action = 1
elif key == "args":
self.args = val
action = 1
elif key == "graceful_timeout":
self.graceful_timeout = float(val)
action = -1
elif key == "max_age":
self.max_age = int(val)
action = 1
elif key == "max_age_variance":
self.max_age_variance = int(val)
action = 1
elif (key.startswith('stdout_stream') or
key.startswith('stderr_stream')):
action = self._reload_stream(key, val)
elif key.startswith('hooks'):
val = val.split(',')
if len(val) == 2:
ignore_error = util.to_bool(val[1])
else:
ignore_error = False
hook = val[0]
self._reload_hook(key, hook, ignore_error)
action = 0
# send update event
self.notify_event("updated", {"time": time.time()})
return action
def get_config(config_file):
if not os.path.exists(config_file):
raise IOError("the configuration file %r does not exist\n" %
config_file)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# reading the global environ first
global_env = dict(os.environ.items())
local_env = dict()
# update environments with [env] section
if 'env' in cfg.sections():
local_env.update(dict(cfg.items('env')))
global_env.update(local_env)
# always set the cfg environment
cfg.set_env(global_env)
# main circus options
config['check_delay'] = dget('circus', 'check_delay', 5., float)
config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
config['endpoint_owner'] = dget('circus', 'endpoint_owner', None, str)
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
DEFAULT_ENDPOINT_SUB)
config['multicast_endpoint'] = dget('circus', 'multicast_endpoint',
DEFAULT_ENDPOINT_MULTICAST)
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None)
config['statsd'] = dget('circus', 'statsd', False, bool)
config['umask'] = dget('circus', 'umask', None)
if config['umask']:
config['umask'] = int(config['umask'], 8)
if config['stats_endpoint'] is None:
config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS
elif not config['statsd']:
warnings.warn(
"You defined a stats_endpoint without "
"setting up statsd to True.", DeprecationWarning)
config['statsd'] = True
config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
config['httpd'] = dget('circus', 'httpd', False, bool)
config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
config['debug'] = dget('circus', 'debug', False, bool)
config['debug_gc'] = dget('circus', 'debug_gc', False, bool)
config['pidfile'] = dget('circus', 'pidfile')
config['loglevel'] = dget('circus', 'loglevel')
config['logoutput'] = dget('circus', 'logoutput')
config['loggerconfig'] = dget('circus', 'loggerconfig', None)
config['fqdn_prefix'] = dget('circus', 'fqdn_prefix', None, str)
config['papa_endpoint'] = dget('circus', 'fqdn_prefix', None, str)
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1].lower()
sock['so_reuseport'] = dget(section, "so_reuseport", False, bool)
sock['replace'] = dget(section, "replace", False, bool)
sockets.append(sock)
if section.startswith("plugin:"):
plugin = dict(cfg.items(section))
plugin['name'] = section
if 'priority' in plugin:
plugin['priority'] = int(plugin['priority'])
plugins.append(plugin)
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section, noreplace=True):
if opt in ('cmd', 'args', 'working_dir', 'uid', 'gid'):
watcher[opt] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
# default bool to False
elif opt in ('shell', 'send_hup', 'stop_children',
'close_child_stderr', 'use_sockets', 'singleton',
'copy_env', 'copy_path', 'close_child_stdout'):
watcher[opt] = dget(section, opt, False, bool)
elif opt == 'stop_signal':
watcher['stop_signal'] = to_signum(val)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timeout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30,
int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = rlimit_value(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_papa' and dget(section, 'use_papa', False,
bool):
if papa:
watcher['use_papa'] = True
else:
warnings.warn(
"Config file says use_papa but the papa "
"module is missing.", ImportWarning)
elif opt.startswith('hooks.'):
hook_name = opt[len('hooks.'):]
val = [elmt.strip() for elmt in val.split(',', 1)]
if len(val) == 1:
val.append(False)
else:
val[1] = to_bool(val[1])
watcher['hooks'][hook_name] = val
# default bool to True
elif opt in ('check_flapping', 'respawn', 'autostart'):
watcher[opt] = dget(section, opt, True, bool)
else:
# freeform
watcher[opt] = val
if watcher['copy_env']:
watcher['env'] = dict(global_env)
else:
watcher['env'] = dict(local_env)
watchers.append(watcher)
# making sure we return consistent lists
sort_by_field(watchers)
sort_by_field(plugins)
sort_by_field(sockets)
# Second pass to make sure env sections apply to all watchers.
def _extend(target, source):
for name, value in source:
if name in target:
continue
target[name] = value
def _expand_vars(target, key, env):
if isinstance(target[key], str):
target[key] = replace_gnu_args(target[key], env=env)
elif isinstance(target[key], dict):
for k in target[key].keys():
_expand_vars(target[key], k, env)
def _expand_section(section, env, exclude=None):
if exclude is None:
exclude = ('name', 'env')
for option in section.keys():
if option in exclude:
continue
_expand_vars(section, option, env)
# build environment for watcher sections
for section in cfg.sections():
if section.startswith('env:'):
section_elements = section.split("env:", 1)[1]
watcher_patterns = [s.strip() for s in section_elements.split(',')]
env_items = dict(cfg.items(section, noreplace=True))
for pattern in watcher_patterns:
match = [w for w in watchers if fnmatch(w['name'], pattern)]
for watcher in match:
watcher['env'].update(env_items)
# expand environment for watcher sections
for watcher in watchers:
env = dict(global_env)
env.update(watcher['env'])
_expand_section(watcher, env)
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
def convert_option(key, val):
if key == "numprocesses":
return int(val)
elif key == "warmup_delay":
return float(val)
elif key == "working_dir":
return val
elif key == "uid":
return val
elif key == "gid":
return val
elif key == "send_hup":
return util.to_bool(val)
elif key == "stop_signal":
return util.to_signum(val)
elif key == "stop_children":
return util.to_bool(val)
elif key == "shell":
return util.to_bool(val)
elif key == "copy_env":
return util.to_bool(val)
elif key == "env":
return util.parse_env_dict(val)
elif key == "cmd":
return val
elif key == "args":
return val
elif key == "retry_in":
return float(val)
elif key == "max_retry":
return int(val)
elif key == "graceful_timeout":
return float(val)
elif key == 'max_age':
return int(val)
elif key == 'max_age_variance':
return int(val)
elif key == 'respawn':
return util.to_bool(val)
elif key == "singleton":
return util.to_bool(val)
elif key.startswith('stderr_stream.') or key.startswith('stdout_stream.'):
subkey = key.split('.', 1)[-1]
if subkey in ('max_bytes', 'backup_count'):
return int(val)
return val
elif key == 'hooks':
res = {}
for hook in val.split(','):
if hook == '':
continue
hook = hook.split(':')
if len(hook) != 2:
raise ArgumentError(hook)
name, value = hook
if name not in _HOOKS:
raise ArgumentError(name)
res[name] = value
return res
elif key.startswith('hooks.'):
# we can also set a single hook
name = key.split('.', 1)[-1]
if name not in _HOOKS:
raise ArgumentError(name)
return val
elif key.startswith('rlimit_'):
return int(val)
raise ArgumentError("unknown key %r" % key)
def convert_option(key, val):
if key == "numprocesses":
return int(val)
elif key == "warmup_delay":
return float(val)
elif key == "working_dir":
return val
elif key == "uid":
return val
elif key == "gid":
return val
elif key == "send_hup":
return util.to_bool(val)
elif key == "stop_signal":
return util.to_signum(val)
elif key == "stop_children":
return util.to_bool(val)
elif key == "shell":
return util.to_bool(val)
elif key == "copy_env":
return util.to_bool(val)
elif key == "env":
return util.parse_env_dict(val)
elif key == "cmd":
return val
elif key == "args":
return val
elif key == "retry_in":
return float(val)
elif key == "max_retry":
return int(val)
elif key == "graceful_timeout":
return float(val)
elif key == 'max_age':
return int(val)
elif key == 'max_age_variance':
return int(val)
elif key == 'respawn':
return util.to_bool(val)
elif key == "singleton":
return util.to_bool(val)
elif key == "close_child_stdin":
return util.to_bool(val)
elif key == "close_child_stdout":
return util.to_bool(val)
elif key == "close_child_stderr":
return util.to_bool(val)
elif key == "use_papa":
return util.to_bool(val)
elif key.startswith('stderr_stream.') or key.startswith('stdout_stream.'):
subkey = key.split('.', 1)[-1]
if subkey in ('max_bytes', 'backup_count'):
return int(val)
return val
elif key == 'hooks':
res = {}
for hook in val.split(','):
if hook == '':
continue
hook = hook.split(':')
if len(hook) != 2:
raise ArgumentError(hook)
name, value = hook
if name not in _HOOKS:
raise ArgumentError(name)
res[name] = value
return res
elif key.startswith('hooks.'):
# we can also set a single hook
name = key.split('.', 1)[-1]
if name not in _HOOKS:
raise ArgumentError(name)
return val
elif key.startswith('rlimit_'):
return int(val)
raise ArgumentError("unknown key %r" % key)
