def test_replace_gnu_args(self):
repl = replace_gnu_args
self.assertEquals('dont change --fd $(circus.me) please',
repl('dont change --fd $(circus.me) please'))
self.assertEquals('thats an int 2',
repl('thats an int $(circus.me)', me=2))
self.assertEquals('foobar',
replace_gnu_args('$(circus.test)', test='foobar'))
self.assertEquals('foobar',
replace_gnu_args('$(circus.test)', test='foobar'))
self.assertEquals(
'foo, foobar, baz',
replace_gnu_args('foo, $(circus.test), baz', test='foobar'))
self.assertEquals(
'foobar',
replace_gnu_args('$(cir.test)', prefix='cir', test='foobar'))
self.assertEquals('thats an int 2',
repl('thats an int $(s.me)', prefix='s', me=2))
self.assertEquals('thats an int 2',
repl('thats an int $(me)', prefix=None, me=2))
def test_replace_gnu_args(self):
repl = replace_gnu_args
self.assertEquals('dont change --fd $(circus.me) please',
repl('dont change --fd $(circus.me) please'))
self.assertEquals('thats an int 2',
repl('thats an int $(circus.me)',
me=2))
self.assertEquals('foobar', replace_gnu_args('$(circus.test)',
test='foobar'))
self.assertEquals('foobar', replace_gnu_args('$(circus.test)',
test='foobar'))
self.assertEquals('foo, foobar, baz',
replace_gnu_args('foo, $(circus.test), baz',
test='foobar'))
self.assertEquals('foobar', replace_gnu_args('$(cir.test)',
prefix='cir',
test='foobar'))
self.assertEquals('thats an int 2',
repl('thats an int $(s.me)', prefix='s',
me=2))
self.assertEquals('thats an int 2',
repl('thats an int $(me)', prefix=None,
me=2))
def format_args(self, sockets_fds=None):
""" It's possible to use environment variables and some other variables
that are available in this context, when spawning the processes.
"""
logger.debug('cmd: ' + bytestring(self.cmd))
logger.debug('args: ' + str(self.args))
current_env = ObjectDict(self.env.copy())
format_kwargs = {
'wid': self.wid,
'shell': self.shell,
'args': self.args,
'env': current_env,
'working_dir': self.working_dir,
'uid': self.uid,
'gid': self.gid,
'rlimits': self.rlimits,
'executable': self.executable,
'use_fds': self.use_fds
}
if sockets_fds is not None:
format_kwargs['sockets'] = sockets_fds
if self.watcher is not None:
for option in self.watcher.optnames:
if option not in format_kwargs\
and hasattr(self.watcher, option):
format_kwargs[option] = getattr(self.watcher, option)
cmd = replace_gnu_args(self.cmd, **format_kwargs)
if '$WID' in cmd or (self.args and '$WID' in self.args):
msg = "Using $WID in the command is deprecated. You should use "\
"the python string format instead. In you case, this means "\
"replacing the $WID in your command by $(WID)."
warnings.warn(msg, DeprecationWarning)
self.cmd = cmd.replace('$WID', str(self.wid))
if self.args is not None:
if isinstance(self.args, string_types):
args = shlex.split(
bytestring(replace_gnu_args(self.args, **format_kwargs)))
else:
args = [
bytestring(replace_gnu_args(arg, **format_kwargs))
for arg in self.args
]
args = shlex.split(bytestring(cmd)) + args
else:
args = shlex.split(bytestring(cmd))
logger.debug("process args: %s", args)
return args
def format_args(self):
""" It's possible to use environment variables and some other variables
that are available in this context, when spawning the processes.
"""
logger.debug('cmd: ' + bytestring(self.cmd))
logger.debug('args: ' + str(self.args))
current_env = ObjectDict(self.env.copy())
format_kwargs = {
'wid': self.wid, 'shell': self.shell, 'args': self.args,
'env': current_env, 'working_dir': self.working_dir,
'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits,
'executable': self.executable, 'use_fds': self.use_fds,
'hostname': socket.gethostname(),
'sockets': self.watcher._get_sockets_fds()}
if self.watcher is not None:
format_kwargs['sockets'] = self.watcher._get_sockets_fds()
for option in self.watcher.optnames:
if option not in format_kwargs\
and hasattr(self.watcher, option):
format_kwargs[option] = getattr(self.watcher, option)
cmd = replace_gnu_args(self.cmd, **format_kwargs)
if '$WID' in cmd or (self.args and '$WID' in self.args):
msg = "Using $WID in the command is deprecated. You should use "\
"the python string format instead. In you case, this means "\
"replacing the $WID in your command by $(WID)."
warnings.warn(msg, DeprecationWarning)
self.cmd = cmd.replace('$WID', str(self.wid))
if self.args is not None:
if isinstance(self.args, string_types):
args = shlex.split(bytestring(replace_gnu_args(
self.args, **format_kwargs)))
else:
args = [bytestring(replace_gnu_args(arg, **format_kwargs))
for arg in self.args]
args = shlex.split(bytestring(cmd)) + args
else:
args = shlex.split(bytestring(cmd))
for k, v in self.env.items():
self.env[k] = replace_gnu_args(v, **format_kwargs)
logger.debug("process args: %s", args)
return args
def spawn_process(self):
"""Spawn process.
Return True if ok, False if the watcher must be stopped
"""
if self.is_stopped():
return True
if not self.call_hook('before_spawn'):
return False
cmd = util.replace_gnu_args(self.cmd, env=self.env)
nb_tries = 0
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
pipe_stdout = self.stdout_redirector is not None
pipe_stderr = self.stderr_redirector is not None
try:
process = Process(self._nextwid, cmd,
args=self.args, working_dir=self.working_dir,
shell=self.shell, uid=self.uid, gid=self.gid,
env=self.env, rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets, watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr)
# stream stderr/stdout if configured
if pipe_stdout and self.stdout_redirector is not None:
self.stdout_redirector.add_redirection('stdout',
process,
process.stdout)
if pipe_stderr and self.stderr_redirector is not None:
self.stderr_redirector.add_redirection('stderr',
process,
process.stderr)
self.processes[process.pid] = process
logger.debug('running %s process [pid %d]', self.name,
process.pid)
if not self.call_hook('after_spawn', pid=process.pid):
self.kill_process(process)
del self.processes[process.pid]
return False
except OSError as e:
logger.warning('error in %r: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {"process_pid": process.pid,
"time": time.time()})
return True
return False
def items(self, section, noreplace=False):
items = StrictConfigParser.items(self, section)
if noreplace:
return items
return [(key, replace_gnu_args(value, env=self._env))
for key, value in items]
def spawn_process(self):
"""Spawn process.
"""
if self.stopped:
return
if not self.call_hook("before_spawn"):
self.stopped = True
return False
cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds(), env=self.env)
self._process_counter += 1
nb_tries = 0
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
pipe_stdout = self.stdout_redirector is not None
pipe_stderr = self.stderr_redirector is not None
try:
process = Process(
self._process_counter,
cmd,
args=self.args,
working_dir=self.working_dir,
shell=self.shell,
uid=self.uid,
gid=self.gid,
env=self.env,
rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets,
watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr,
)
# stream stderr/stdout if configured
if pipe_stdout:
self.stdout_redirector.add_redirection("stdout", process, process.stdout)
if pipe_stderr:
self.stderr_redirector.add_redirection("stderr", process, process.stderr)
self.processes[process.pid] = process
logger.debug("running %s process [pid %d]", self.name, process.pid)
except OSError as e:
logger.warning("error in %r: %s", self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()})
time.sleep(self.warmup_delay)
return
self.stop()
def items(self, section, noreplace=False):
items = StrictConfigParser.items(self, section)
if noreplace:
return items
return [(key, replace_gnu_args(value, env=self._env))
for key, value in items]
def spawn_process(self):
"""Spawn process.
"""
if self.stopped:
return
cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds())
self._process_counter += 1
nb_tries = 0
pipe_stdout = self.stdout_redirector is not None
pipe_stderr = self.stderr_redirector is not None
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
try:
process = Process(self._process_counter,
cmd,
args=self.args,
working_dir=self.working_dir,
shell=self.shell,
uid=self.uid,
gid=self.gid,
env=self.env,
rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets,
watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr)
# stream stderr/stdout if configured
if pipe_stdout:
self.stdout_redirector.add_redirection(
'stdout', process, process.stdout)
if pipe_stderr:
self.stderr_redirector.add_redirection(
'stderr', process, process.stderr)
self.processes[process.pid] = process
logger.debug('running %s process [pid %d]', self.name,
process.pid)
except OSError, e:
logger.warning('error in %r: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {
"process_pid": process.pid,
"time": time.time()
})
time.sleep(self.warmup_delay)
return
def spawn_process(self):
"""Spawn process.
Return True if ok, False if the watcher must be stopped
"""
if self.is_stopped():
return True
if not self.call_hook('before_spawn'):
return False
cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds(),
env=self.env)
nb_tries = 0
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
pipe_stdout = self.stdout_redirector is not None
pipe_stderr = self.stderr_redirector is not None
try:
process = Process(self._nextwid, cmd,
args=self.args, working_dir=self.working_dir,
shell=self.shell, uid=self.uid, gid=self.gid,
env=self.env, rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets, watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr)
# stream stderr/stdout if configured
if pipe_stdout and self.stdout_redirector is not None:
self.stdout_redirector.add_redirection('stdout',
process,
process.stdout)
if pipe_stderr and self.stderr_redirector is not None:
self.stderr_redirector.add_redirection('stderr',
process,
process.stderr)
self.processes[process.pid] = process
logger.debug('running %s process [pid %d]', self.name,
process.pid)
except OSError as e:
logger.warning('error in %r: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {"process_pid": process.pid,
"time": time.time()})
return True
return False
def test_replace_gnu_args(self):
repl = replace_gnu_args
self.assertEquals("dont change --fd ((circus.me)) please", repl("dont change --fd ((circus.me)) please"))
self.assertEquals("dont change --fd $(circus.me) please", repl("dont change --fd $(circus.me) please"))
self.assertEquals("thats an int 2", repl("thats an int $(circus.me)", me=2))
self.assertEquals("foobar", replace_gnu_args("$(circus.test)", test="foobar"))
self.assertEquals("foobar", replace_gnu_args("$(circus.test)", test="foobar"))
self.assertEquals("foo, foobar, baz", replace_gnu_args("foo, $(circus.test), baz", test="foobar"))
self.assertEquals("foo, foobar, baz", replace_gnu_args("foo, ((circus.test)), baz", test="foobar"))
self.assertEquals("foobar", replace_gnu_args("$(cir.test)", prefix="cir", test="foobar"))
self.assertEquals("foobar", replace_gnu_args("((cir.test))", prefix="cir", test="foobar"))
self.assertEquals("thats an int 2", repl("thats an int $(s.me)", prefix="s", me=2))
self.assertEquals("thats an int 2", repl("thats an int ((s.me))", prefix="s", me=2))
self.assertEquals("thats an int 2", repl("thats an int $(me)", prefix=None, me=2))
self.assertEquals("thats an int 2", repl("thats an int ((me))", prefix=None, me=2))
def spawn_process(self):
"""Spawn process.
"""
if self.stopped:
return
cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds())
self._process_counter += 1
nb_tries = 0
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
try:
process = Process(self._process_counter, cmd,
args=self.args, working_dir=self.working_dir,
shell=self.shell, uid=self.uid, gid=self.gid,
env=self.env, rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets, watcher=self,
_exec=self._exec)
# stream stderr/stdout if configured
if self.stdout_redirector is not None:
self.stdout_redirector.add_redirection('stdout',
process,
process.stdout)
if self.stderr_redirector is not None:
self.stderr_redirector.add_redirection('stderr',
process,
process.stderr)
self.processes[process.pid] = process
logger.debug('running %s process [pid %d]', self.name,
process.pid)
except OSError, e:
logger.warning('error in %r: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {"process_pid": process.pid,
"time": time.time()})
time.sleep(self.warmup_delay)
return
def _expand_vars(target, key, env):
if isinstance(target[key], str):
target[key] = replace_gnu_args(target[key], env=env)
elif isinstance(target[key], dict):
for k in target[key].keys():
_expand_vars(target[key], k, env)
def format_args(self, sockets_fds=None):
""" It's possible to use environment variables and some other variables
that are available in this context, when spawning the processes.
"""
logger.debug('cmd: ' + bytestring(self.cmd))
logger.debug('args: ' + str(self.args))
current_env = ObjectDict(self.env.copy())
format_kwargs = {
'wid': self.wid,
'shell': self.shell,
'args': self.args,
'env': current_env,
'working_dir': self.working_dir,
'uid': self.uid,
'gid': self.gid,
'rlimits': self.rlimits,
'executable': self.executable,
'use_fds': self.use_fds
}
if sockets_fds is not None:
format_kwargs['sockets'] = sockets_fds
if self.watcher is not None:
for option in self.watcher.optnames:
if option not in format_kwargs\
and hasattr(self.watcher, option):
format_kwargs[option] = getattr(self.watcher, option)
cmd = replace_gnu_args(self.cmd, **format_kwargs)
if '$WID' in cmd or (self.args and '$WID' in self.args):
msg = "Using $WID in the command is deprecated. You should use "\
"the python string format instead. In you case, this means "\
"replacing the $WID in your command by $(WID)."
warnings.warn(msg, DeprecationWarning)
self.cmd = cmd.replace('$WID', str(self.wid))
if self.args is not None:
if isinstance(self.args, string_types):
args = shlex.split(
bytestring(replace_gnu_args(self.args, **format_kwargs)))
else:
args = [
bytestring(replace_gnu_args(arg, **format_kwargs))
for arg in self.args
]
args = shlex.split(bytestring(cmd)) + args
else:
args = shlex.split(bytestring(cmd))
if self.shell:
# subprocess.Popen(shell=True) implies that 1st arg is the
# requested command, remaining args are applied to sh.
args = [' '.join(quote(arg) for arg in args)]
shell_args = format_kwargs.get('shell_args', None)
if shell_args and is_win():
logger.warn(
"shell_args won't apply for "
"windows platforms: %s", shell_args)
elif isinstance(shell_args, string_types):
args += shlex.split(
bytestring(replace_gnu_args(shell_args, **format_kwargs)))
elif shell_args:
args += [
bytestring(replace_gnu_args(arg, **format_kwargs))
for arg in shell_args
]
elif format_kwargs.get('shell_args', False):
logger.warn(
"shell_args is defined but won't be used "
"in this context: %s", format_kwargs['shell_args'])
logger.debug("process args: %s", args)
return args
def spawn_process(self, recovery_wid=None):
"""Spawn process.
Return True if ok, False if the watcher must be stopped
"""
if self.is_stopped():
return True
if not recovery_wid and not self.call_hook('before_spawn'):
return False
cmd = util.replace_gnu_args(self.cmd, env=self.env)
nb_tries = 0
# start the redirector now so we can catch any startup errors
if self.stream_redirector:
self.stream_redirector.start()
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
pipe_stdout = self.stdout_stream is not None
pipe_stderr = self.stderr_stream is not None
# noinspection PyPep8Naming
ProcCls = self._process_class
try:
process = ProcCls(self.name, recovery_wid or self._nextwid,
cmd, args=self.args,
working_dir=self.working_dir,
shell=self.shell, uid=self.uid, gid=self.gid,
env=self.env, rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets, watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdin=self.close_child_stdin,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr)
# stream stderr/stdout if configured
if self.stream_redirector:
self.stream_redirector.add_redirections(process)
self.processes[process.pid] = process
logger.debug('running %s process [pid %d]', self.name,
process.pid)
if not self.call_hook('after_spawn', pid=process.pid):
self.kill_process(process)
del self.processes[process.pid]
return False
# catch ValueError as well, as a misconfigured rlimit setting could
# lead to bad infinite retries here
except (OSError, ValueError) as e:
logger.warning('error in %r: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {"process_pid": process.pid,
"time": process.started})
return process.started
return False
def get(self, section, option):
res = StrictConfigParser.get(self, section, option)
return replace_gnu_args(res, env=self._env)
def _expand_vars(target, key, env):
if isinstance(target[key], str):
target[key] = replace_gnu_args(target[key], env=env)
elif isinstance(target[key], dict):
for k in target[key].keys():
_expand_vars(target[key], k, env)
def test_dashes(self):
conf = get_config(_CONF["issue546"])
replaced = replace_gnu_args(conf["watchers"][0]["cmd"], sockets={"some-socket": 3})
self.assertEqual(replaced, "../bin/chaussette --fd 3")
def get(self, section, option):
res = StrictConfigParser.get(self, section, option)
return replace_gnu_args(res, env=self._env)
def test_dashes(self):
conf = get_config(_CONF['issue546'])
replaced = replace_gnu_args(conf['watchers'][0]['cmd'],
sockets={'some-socket': 3})
self.assertEqual(replaced, '../bin/chaussette --fd 3')
def get_config(config_file):
if not os.path.exists(config_file):
raise IOError("the configuration file %r does not exist\n" %
config_file)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# reading the global environ first
def _upper(items):
return [(key.upper(), value) for key, value in items]
global_env = dict(_upper(os.environ.items()))
if 'env' in cfg.sections():
global_env.update(dict(_upper(cfg.items('env'))))
cfg.set_env(global_env)
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
DEFAULT_ENDPOINT_SUB)
config['multicast_endpoint'] = dget('circus', 'multicast_endpoint',
DEFAULT_ENDPOINT_MULTICAST)
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None)
config['statsd'] = dget('circus', 'statsd', False, bool)
if config['stats_endpoint'] is None:
config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS
elif not config['statsd']:
warnings.warn("You defined a stats_endpoint without "
"setting up statsd to True.",
DeprecationWarning)
config['statsd'] = True
config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
config['httpd'] = dget('circus', 'httpd', False, bool)
config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
config['debug'] = dget('circus', 'debug', False, bool)
config['pidfile'] = dget('circus', 'pidfile')
config['loglevel'] = dget('circus', 'loglevel')
config['logoutput'] = dget('circus', 'logoutput')
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1].lower()
sockets.append(sock)
if section.startswith("plugin:"):
plugin = dict(cfg.items(section))
plugin['name'] = section
plugins.append(plugin)
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section, noreplace=True):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timeout':
watcher['graceful_timeout'] = dget(
section, "graceful_timeout", 30, int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
elif opt == 'singleton':
watcher['singleton'] = dget(section, "singleton", False,
bool)
elif opt == 'copy_env':
watcher['copy_env'] = dget(section, "copy_env", False,
bool)
elif opt == 'copy_path':
watcher['copy_path'] = dget(section, "copy_path", False,
bool)
elif opt.startswith('hooks.'):
hook_name = opt[len('hooks.'):]
val = [elmt.strip() for elmt in val.split(',', 1)]
if len(val) == 1:
val.append(False)
else:
val[1] = to_boolean(val[1])
watcher['hooks'][hook_name] = val
elif opt == 'respawn':
watcher['respawn'] = dget(section, "respawn", True, bool)
elif opt == 'env':
logger.warning('the env option is deprecated the use of '
'env sections is recommended')
watcher['env'] = parse_env_str(val)
elif opt == 'autostart':
watcher['autostart'] = dget(section, "autostart", True,
bool)
elif opt == 'close_child_stdout':
watcher['close_child_stdout'] = dget(section,
"close_child_stdout",
False, bool)
elif opt == 'close_child_stderr':
watcher['close_child_stderr'] = dget(section,
"close_child_stderr",
False, bool)
else:
# freeform
watcher[opt] = val
watchers.append(watcher)
# Second pass to make sure env sections apply to all watchers.
environs = defaultdict(dict)
# global env first
def _extend(target, source):
for name, value in source:
if name in target:
continue
target[name] = value
for watcher in watchers:
_extend(environs[watcher['name']], global_env.items())
# then per-watcher env
for section in cfg.sections():
if section.startswith('env:'):
section_elements = section.split("env:", 1)[1]
watcher_patterns = [s.strip() for s in section_elements.split(',')]
for pattern in watcher_patterns:
match = [w for w in watchers if fnmatch(w['name'], pattern)]
for watcher in match:
watcher_name = watcher['name']
extra = cfg.items(section, noreplace=True)
environs[watcher_name].update(_upper(extra))
for watcher in watchers:
if watcher['name'] in environs:
if not 'env' in watcher:
watcher['env'] = dict()
_extend(watcher['env'], environs[watcher['name']].items())
for option, value in watcher.items():
if option in ('name', 'env'):
continue
if not isinstance(value, str):
continue
watcher[option] = replace_gnu_args(value, env=watcher['env'])
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config
def format_args(self, sockets_fds=None):
""" It's possible to use environment variables and some other variables
that are available in this context, when spawning the processes.
"""
logger.debug('cmd: ' + bytestring(self.cmd))
logger.debug('args: ' + str(self.args))
current_env = ObjectDict(self.env.copy())
format_kwargs = {
'wid': self.wid, 'shell': self.shell, 'args': self.args,
'env': current_env, 'working_dir': self.working_dir,
'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits,
'executable': self.executable, 'use_fds': self.use_fds}
if sockets_fds is not None:
format_kwargs['sockets'] = sockets_fds
if self.watcher is not None:
for option in self.watcher.optnames:
if option not in format_kwargs\
and hasattr(self.watcher, option):
format_kwargs[option] = getattr(self.watcher, option)
cmd = replace_gnu_args(self.cmd, **format_kwargs)
if '$WID' in cmd or (self.args and '$WID' in self.args):
msg = "Using $WID in the command is deprecated. You should use "\
"the python string format instead. In you case, this means "\
"replacing the $WID in your command by $(WID)."
warnings.warn(msg, DeprecationWarning)
self.cmd = cmd.replace('$WID', str(self.wid))
if self.args is not None:
if isinstance(self.args, string_types):
args = shlex.split(bytestring(replace_gnu_args(
self.args, **format_kwargs)))
else:
args = [bytestring(replace_gnu_args(arg, **format_kwargs))
for arg in self.args]
args = shlex.split(bytestring(cmd)) + args
else:
args = shlex.split(bytestring(cmd))
if self.shell:
# subprocess.Popen(shell=True) implies that 1st arg is the
# requested command, remaining args are applied to sh.
args = [' '.join(quote(arg) for arg in args)]
shell_args = format_kwargs.get('shell_args', None)
if shell_args and is_win():
logger.warn("shell_args won't apply for "
"windows platforms: %s", shell_args)
elif isinstance(shell_args, string_types):
args += shlex.split(bytestring(replace_gnu_args(
shell_args, **format_kwargs)))
elif shell_args:
args += [bytestring(replace_gnu_args(arg, **format_kwargs))
for arg in shell_args]
elif format_kwargs.get('shell_args', False):
logger.warn("shell_args is defined but won't be used "
"in this context: %s", format_kwargs['shell_args'])
logger.debug("process args: %s", args)
return args
def spawn_process(self, recovery_wid=None):
"""Spawn process.
Return True if ok, False if the watcher must be stopped
"""
if self.is_stopped():
return True
if not recovery_wid and not self.call_hook('before_spawn'):
return False
cmd = util.replace_gnu_args(self.cmd, env=self.env)
nb_tries = 0
# start the redirector now so we can catch any startup errors
if self.stream_redirector:
self.stream_redirector.start()
while nb_tries < self.max_retry or self.max_retry == -1:
process = None
pipe_stdout = self.stdout_stream is not None
pipe_stderr = self.stderr_stream is not None
# noinspection PyPep8Naming
ProcCls = self._process_class
try:
process = ProcCls(self.name,
recovery_wid or self._nextwid,
cmd,
args=self.args,
working_dir=self.working_dir,
shell=self.shell,
uid=self.uid,
gid=self.gid,
env=self.env,
rlimits=self.rlimits,
executable=self.executable,
use_fds=self.use_sockets,
watcher=self,
pipe_stdout=pipe_stdout,
pipe_stderr=pipe_stderr,
close_child_stdin=self.close_child_stdin,
close_child_stdout=self.close_child_stdout,
close_child_stderr=self.close_child_stderr)
# stream stderr/stdout if configured
if self.stream_redirector:
self.stream_redirector.add_redirections(process)
self.processes[process.pid] = process
logger.debug('running %s process [pid %d]', self.name,
process.pid)
if not self.call_hook('after_spawn', pid=process.pid):
self.kill_process(process)
del self.processes[process.pid]
return False
# catch ValueError as well, as a misconfigured rlimit setting could
# lead to bad infinite retries here
except (OSError, ValueError) as e:
logger.warning('error in %r: %s', self.name, str(e))
if process is None:
nb_tries += 1
continue
else:
self.notify_event("spawn", {
"process_pid": process.pid,
"time": process.started
})
return process.started
return False
def get_config(config_file):
if not os.path.exists(config_file):
raise IOError("the configuration file %r does not exist\n" %
config_file)
cfg, cfg_files_read = read_config(config_file)
dget = cfg.dget
config = {}
# reading the global environ first
def _upper(items):
return [(key.upper(), value) for key, value in items]
global_env = dict(_upper(os.environ.items()))
if 'env' in cfg.sections():
global_env.update(dict(_upper(cfg.items('env'))))
cfg.set_env(global_env)
# main circus options
config['check'] = dget('circus', 'check_delay', 5, int)
config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER)
config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint',
DEFAULT_ENDPOINT_SUB)
config['multicast_endpoint'] = dget('circus', 'multicast_endpoint',
DEFAULT_ENDPOINT_MULTICAST)
config['stats_endpoint'] = dget('circus', 'stats_endpoint', None)
config['statsd'] = dget('circus', 'statsd', False, bool)
if config['stats_endpoint'] is None:
config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS
elif not config['statsd']:
warnings.warn(
"You defined a stats_endpoint without "
"setting up statsd to True.", DeprecationWarning)
config['statsd'] = True
config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int)
config['httpd'] = dget('circus', 'httpd', False, bool)
config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str)
config['httpd_port'] = dget('circus', 'httpd_port', 8080, int)
config['debug'] = dget('circus', 'debug', False, bool)
config['pidfile'] = dget('circus', 'pidfile')
config['loglevel'] = dget('circus', 'loglevel')
config['logoutput'] = dget('circus', 'logoutput')
config['fqdn_prefix'] = dget('circus', 'fqdn_prefix', None, str)
# Initialize watchers, plugins & sockets to manage
watchers = []
plugins = []
sockets = []
for section in cfg.sections():
if section.startswith("socket:"):
sock = dict(cfg.items(section))
sock['name'] = section.split("socket:")[-1].lower()
sockets.append(sock)
if section.startswith("plugin:"):
plugin = dict(cfg.items(section))
plugin['name'] = section
plugins.append(plugin)
if section.startswith("watcher:"):
watcher = watcher_defaults()
watcher['name'] = section.split("watcher:", 1)[1]
# create watcher options
for opt, val in cfg.items(section, noreplace=True):
if opt == 'cmd':
watcher['cmd'] = val
elif opt == 'args':
watcher['args'] = val
elif opt == 'numprocesses':
watcher['numprocesses'] = dget(section, 'numprocesses', 1,
int)
elif opt == 'warmup_delay':
watcher['warmup_delay'] = dget(section, 'warmup_delay', 0,
int)
elif opt == 'executable':
watcher['executable'] = dget(section, 'executable', None,
str)
elif opt == 'working_dir':
watcher['working_dir'] = val
elif opt == 'shell':
watcher['shell'] = dget(section, 'shell', False, bool)
elif opt == 'uid':
watcher['uid'] = val
elif opt == 'gid':
watcher['gid'] = val
elif opt == 'send_hup':
watcher['send_hup'] = dget(section, 'send_hup', False,
bool)
elif opt == 'check_flapping':
watcher['check_flapping'] = dget(section, 'check_flapping',
True, bool)
elif opt == 'max_retry':
watcher['max_retry'] = dget(section, "max_retry", 5, int)
elif opt == 'graceful_timeout':
watcher['graceful_timeout'] = dget(section,
"graceful_timeout", 30,
int)
elif opt.startswith('stderr_stream') or \
opt.startswith('stdout_stream'):
stream_name, stream_opt = opt.split(".", 1)
watcher[stream_name][stream_opt] = val
elif opt.startswith('rlimit_'):
limit = opt[7:]
watcher['rlimits'][limit] = int(val)
elif opt == 'priority':
watcher['priority'] = dget(section, "priority", 0, int)
elif opt == 'use_sockets':
watcher['use_sockets'] = dget(section, "use_sockets",
False, bool)
elif opt == 'singleton':
watcher['singleton'] = dget(section, "singleton", False,
bool)
elif opt == 'copy_env':
watcher['copy_env'] = dget(section, "copy_env", False,
bool)
elif opt == 'copy_path':
watcher['copy_path'] = dget(section, "copy_path", False,
bool)
elif opt.startswith('hooks.'):
hook_name = opt[len('hooks.'):]
val = [elmt.strip() for elmt in val.split(',', 1)]
if len(val) == 1:
val.append(False)
else:
val[1] = to_boolean(val[1])
watcher['hooks'][hook_name] = val
elif opt == 'respawn':
watcher['respawn'] = dget(section, "respawn", True, bool)
elif opt == 'env':
logger.warning('the env option is deprecated the use of '
'env sections is recommended')
watcher['env'] = parse_env_str(val)
elif opt == 'autostart':
watcher['autostart'] = dget(section, "autostart", True,
bool)
elif opt == 'close_child_stdout':
watcher['close_child_stdout'] = dget(
section, "close_child_stdout", False, bool)
elif opt == 'close_child_stderr':
watcher['close_child_stderr'] = dget(
section, "close_child_stderr", False, bool)
else:
# freeform
watcher[opt] = val
watchers.append(watcher)
# Second pass to make sure env sections apply to all watchers.
environs = defaultdict(dict)
# global env first
def _extend(target, source):
for name, value in source:
if name in target:
continue
target[name] = value
for watcher in watchers:
_extend(environs[watcher['name']], global_env.items())
# then per-watcher env
for section in cfg.sections():
if section.startswith('env:'):
section_elements = section.split("env:", 1)[1]
watcher_patterns = [s.strip() for s in section_elements.split(',')]
for pattern in watcher_patterns:
match = [w for w in watchers if fnmatch(w['name'], pattern)]
for watcher in match:
watcher_name = watcher['name']
extra = cfg.items(section, noreplace=True)
environs[watcher_name].update(_upper(extra))
for watcher in watchers:
if watcher['name'] in environs:
if not 'env' in watcher:
watcher['env'] = dict()
_extend(watcher['env'], environs[watcher['name']].items())
for option, value in watcher.items():
if option in ('name', 'env'):
continue
if not isinstance(value, str):
continue
watcher[option] = replace_gnu_args(value, env=watcher['env'])
config['watchers'] = watchers
config['plugins'] = plugins
config['sockets'] = sockets
return config