def test_replace_gnu_args(self): repl = replace_gnu_args self.assertEquals('dont change --fd $(circus.me) please', repl('dont change --fd $(circus.me) please')) self.assertEquals('thats an int 2', repl('thats an int $(circus.me)', me=2)) self.assertEquals('foobar', replace_gnu_args('$(circus.test)', test='foobar')) self.assertEquals('foobar', replace_gnu_args('$(circus.test)', test='foobar')) self.assertEquals( 'foo, foobar, baz', replace_gnu_args('foo, $(circus.test), baz', test='foobar')) self.assertEquals( 'foobar', replace_gnu_args('$(cir.test)', prefix='cir', test='foobar')) self.assertEquals('thats an int 2', repl('thats an int $(s.me)', prefix='s', me=2)) self.assertEquals('thats an int 2', repl('thats an int $(me)', prefix=None, me=2))
def test_replace_gnu_args(self): repl = replace_gnu_args self.assertEquals('dont change --fd $(circus.me) please', repl('dont change --fd $(circus.me) please')) self.assertEquals('thats an int 2', repl('thats an int $(circus.me)', me=2)) self.assertEquals('foobar', replace_gnu_args('$(circus.test)', test='foobar')) self.assertEquals('foobar', replace_gnu_args('$(circus.test)', test='foobar')) self.assertEquals('foo, foobar, baz', replace_gnu_args('foo, $(circus.test), baz', test='foobar')) self.assertEquals('foobar', replace_gnu_args('$(cir.test)', prefix='cir', test='foobar')) self.assertEquals('thats an int 2', repl('thats an int $(s.me)', prefix='s', me=2)) self.assertEquals('thats an int 2', repl('thats an int $(me)', prefix=None, me=2))
def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds } if sockets_fds is not None: format_kwargs['sockets'] = sockets_fds if self.watcher is not None: for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split( bytestring(replace_gnu_args(self.args, **format_kwargs))) else: args = [ bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args ] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) logger.debug("process args: %s", args) return args
def format_args(self): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds, 'hostname': socket.gethostname(), 'sockets': self.watcher._get_sockets_fds()} if self.watcher is not None: format_kwargs['sockets'] = self.watcher._get_sockets_fds() for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split(bytestring(replace_gnu_args( self.args, **format_kwargs))) else: args = [bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) for k, v in self.env.items(): self.env[k] = replace_gnu_args(v, **format_kwargs) logger.debug("process args: %s", args) return args
def spawn_process(self): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, env=self.env) nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None try: process = Process(self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if pipe_stdout and self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if pipe_stderr and self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) if not self.call_hook('after_spawn', pid=process.pid): self.kill_process(process) del self.processes[process.pid] return False except OSError as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) return True return False
def items(self, section, noreplace=False): items = StrictConfigParser.items(self, section) if noreplace: return items return [(key, replace_gnu_args(value, env=self._env)) for key, value in items]
def spawn_process(self): """Spawn process. """ if self.stopped: return if not self.call_hook("before_spawn"): self.stopped = True return False cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds(), env=self.env) self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None try: process = Process( self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr, ) # stream stderr/stdout if configured if pipe_stdout: self.stdout_redirector.add_redirection("stdout", process, process.stdout) if pipe_stderr: self.stderr_redirector.add_redirection("stderr", process, process.stderr) self.processes[process.pid] = process logger.debug("running %s process [pid %d]", self.name, process.pid) except OSError as e: logger.warning("error in %r: %s", self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return self.stop()
def spawn_process(self): """Spawn process. """ if self.stopped: return cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds()) self._process_counter += 1 nb_tries = 0 pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None while nb_tries < self.max_retry or self.max_retry == -1: process = None try: process = Process(self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if pipe_stdout: self.stdout_redirector.add_redirection( 'stdout', process, process.stdout) if pipe_stderr: self.stderr_redirector.add_redirection( 'stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", { "process_pid": process.pid, "time": time.time() }) time.sleep(self.warmup_delay) return
def spawn_process(self): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds(), env=self.env) nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None try: process = Process(self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if pipe_stdout and self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if pipe_stderr and self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) return True return False
def test_replace_gnu_args(self): repl = replace_gnu_args self.assertEquals("dont change --fd ((circus.me)) please", repl("dont change --fd ((circus.me)) please")) self.assertEquals("dont change --fd $(circus.me) please", repl("dont change --fd $(circus.me) please")) self.assertEquals("thats an int 2", repl("thats an int $(circus.me)", me=2)) self.assertEquals("foobar", replace_gnu_args("$(circus.test)", test="foobar")) self.assertEquals("foobar", replace_gnu_args("$(circus.test)", test="foobar")) self.assertEquals("foo, foobar, baz", replace_gnu_args("foo, $(circus.test), baz", test="foobar")) self.assertEquals("foo, foobar, baz", replace_gnu_args("foo, ((circus.test)), baz", test="foobar")) self.assertEquals("foobar", replace_gnu_args("$(cir.test)", prefix="cir", test="foobar")) self.assertEquals("foobar", replace_gnu_args("((cir.test))", prefix="cir", test="foobar")) self.assertEquals("thats an int 2", repl("thats an int $(s.me)", prefix="s", me=2)) self.assertEquals("thats an int 2", repl("thats an int ((s.me))", prefix="s", me=2)) self.assertEquals("thats an int 2", repl("thats an int $(me)", prefix=None, me=2)) self.assertEquals("thats an int 2", repl("thats an int ((me))", prefix=None, me=2))
def spawn_process(self): """Spawn process. """ if self.stopped: return cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds()) self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None try: process = Process(self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, _exec=self._exec) # stream stderr/stdout if configured if self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return
def _expand_vars(target, key, env): if isinstance(target[key], str): target[key] = replace_gnu_args(target[key], env=env) elif isinstance(target[key], dict): for k in target[key].keys(): _expand_vars(target[key], k, env)
def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds } if sockets_fds is not None: format_kwargs['sockets'] = sockets_fds if self.watcher is not None: for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split( bytestring(replace_gnu_args(self.args, **format_kwargs))) else: args = [ bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args ] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) if self.shell: # subprocess.Popen(shell=True) implies that 1st arg is the # requested command, remaining args are applied to sh. args = [' '.join(quote(arg) for arg in args)] shell_args = format_kwargs.get('shell_args', None) if shell_args and is_win(): logger.warn( "shell_args won't apply for " "windows platforms: %s", shell_args) elif isinstance(shell_args, string_types): args += shlex.split( bytestring(replace_gnu_args(shell_args, **format_kwargs))) elif shell_args: args += [ bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in shell_args ] elif format_kwargs.get('shell_args', False): logger.warn( "shell_args is defined but won't be used " "in this context: %s", format_kwargs['shell_args']) logger.debug("process args: %s", args) return args
def spawn_process(self, recovery_wid=None): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not recovery_wid and not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, env=self.env) nb_tries = 0 # start the redirector now so we can catch any startup errors if self.stream_redirector: self.stream_redirector.start() while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_stream is not None pipe_stderr = self.stderr_stream is not None # noinspection PyPep8Naming ProcCls = self._process_class try: process = ProcCls(self.name, recovery_wid or self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdin=self.close_child_stdin, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if self.stream_redirector: self.stream_redirector.add_redirections(process) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) if not self.call_hook('after_spawn', pid=process.pid): self.kill_process(process) del self.processes[process.pid] return False # catch ValueError as well, as a misconfigured rlimit setting could # lead to bad infinite retries here except (OSError, ValueError) as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": process.started}) return process.started return False
def get(self, section, option): res = StrictConfigParser.get(self, section, option) return replace_gnu_args(res, env=self._env)
def test_dashes(self): conf = get_config(_CONF["issue546"]) replaced = replace_gnu_args(conf["watchers"][0]["cmd"], sockets={"some-socket": 3}) self.assertEqual(replaced, "../bin/chaussette --fd 3")
def test_dashes(self): conf = get_config(_CONF['issue546']) replaced = replace_gnu_args(conf['watchers'][0]['cmd'], sockets={'some-socket': 3}) self.assertEqual(replaced, '../bin/chaussette --fd 3')
def get_config(config_file): if not os.path.exists(config_file): raise IOError("the configuration file %r does not exist\n" % config_file) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # reading the global environ first def _upper(items): return [(key.upper(), value) for key, value in items] global_env = dict(_upper(os.environ.items())) if 'env' in cfg.sections(): global_env.update(dict(_upper(cfg.items('env')))) cfg.set_env(global_env) # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['multicast_endpoint'] = dget('circus', 'multicast_endpoint', DEFAULT_ENDPOINT_MULTICAST) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None) config['statsd'] = dget('circus', 'statsd', False, bool) if config['stats_endpoint'] is None: config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS elif not config['statsd']: warnings.warn("You defined a stats_endpoint without " "setting up statsd to True.", DeprecationWarning) config['statsd'] = True config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) config['pidfile'] = dget('circus', 'pidfile') config['loglevel'] = dget('circus', 'loglevel') config['logoutput'] = dget('circus', 'logoutput') # Initialize watchers, plugins & sockets to manage watchers = [] plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugin = dict(cfg.items(section)) plugin['name'] = section plugins.append(plugin) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section, noreplace=True): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timeout': watcher['graceful_timeout'] = dget( section, "graceful_timeout", 30, int) elif opt.startswith('stderr_stream') or \ opt.startswith('stdout_stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) elif opt == 'autostart': watcher['autostart'] = dget(section, "autostart", True, bool) elif opt == 'close_child_stdout': watcher['close_child_stdout'] = dget(section, "close_child_stdout", False, bool) elif opt == 'close_child_stderr': watcher['close_child_stderr'] = dget(section, "close_child_stderr", False, bool) else: # freeform watcher[opt] = val watchers.append(watcher) # Second pass to make sure env sections apply to all watchers. environs = defaultdict(dict) # global env first def _extend(target, source): for name, value in source: if name in target: continue target[name] = value for watcher in watchers: _extend(environs[watcher['name']], global_env.items()) # then per-watcher env for section in cfg.sections(): if section.startswith('env:'): section_elements = section.split("env:", 1)[1] watcher_patterns = [s.strip() for s in section_elements.split(',')] for pattern in watcher_patterns: match = [w for w in watchers if fnmatch(w['name'], pattern)] for watcher in match: watcher_name = watcher['name'] extra = cfg.items(section, noreplace=True) environs[watcher_name].update(_upper(extra)) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() _extend(watcher['env'], environs[watcher['name']].items()) for option, value in watcher.items(): if option in ('name', 'env'): continue if not isinstance(value, str): continue watcher[option] = replace_gnu_args(value, env=watcher['env']) config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config
def format_args(self, sockets_fds=None): """ It's possible to use environment variables and some other variables that are available in this context, when spawning the processes. """ logger.debug('cmd: ' + bytestring(self.cmd)) logger.debug('args: ' + str(self.args)) current_env = ObjectDict(self.env.copy()) format_kwargs = { 'wid': self.wid, 'shell': self.shell, 'args': self.args, 'env': current_env, 'working_dir': self.working_dir, 'uid': self.uid, 'gid': self.gid, 'rlimits': self.rlimits, 'executable': self.executable, 'use_fds': self.use_fds} if sockets_fds is not None: format_kwargs['sockets'] = sockets_fds if self.watcher is not None: for option in self.watcher.optnames: if option not in format_kwargs\ and hasattr(self.watcher, option): format_kwargs[option] = getattr(self.watcher, option) cmd = replace_gnu_args(self.cmd, **format_kwargs) if '$WID' in cmd or (self.args and '$WID' in self.args): msg = "Using $WID in the command is deprecated. You should use "\ "the python string format instead. In you case, this means "\ "replacing the $WID in your command by $(WID)." warnings.warn(msg, DeprecationWarning) self.cmd = cmd.replace('$WID', str(self.wid)) if self.args is not None: if isinstance(self.args, string_types): args = shlex.split(bytestring(replace_gnu_args( self.args, **format_kwargs))) else: args = [bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in self.args] args = shlex.split(bytestring(cmd)) + args else: args = shlex.split(bytestring(cmd)) if self.shell: # subprocess.Popen(shell=True) implies that 1st arg is the # requested command, remaining args are applied to sh. args = [' '.join(quote(arg) for arg in args)] shell_args = format_kwargs.get('shell_args', None) if shell_args and is_win(): logger.warn("shell_args won't apply for " "windows platforms: %s", shell_args) elif isinstance(shell_args, string_types): args += shlex.split(bytestring(replace_gnu_args( shell_args, **format_kwargs))) elif shell_args: args += [bytestring(replace_gnu_args(arg, **format_kwargs)) for arg in shell_args] elif format_kwargs.get('shell_args', False): logger.warn("shell_args is defined but won't be used " "in this context: %s", format_kwargs['shell_args']) logger.debug("process args: %s", args) return args
def spawn_process(self, recovery_wid=None): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not recovery_wid and not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, env=self.env) nb_tries = 0 # start the redirector now so we can catch any startup errors if self.stream_redirector: self.stream_redirector.start() while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_stream is not None pipe_stderr = self.stderr_stream is not None # noinspection PyPep8Naming ProcCls = self._process_class try: process = ProcCls(self.name, recovery_wid or self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdin=self.close_child_stdin, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if self.stream_redirector: self.stream_redirector.add_redirections(process) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) if not self.call_hook('after_spawn', pid=process.pid): self.kill_process(process) del self.processes[process.pid] return False # catch ValueError as well, as a misconfigured rlimit setting could # lead to bad infinite retries here except (OSError, ValueError) as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", { "process_pid": process.pid, "time": process.started }) return process.started return False
def get_config(config_file): if not os.path.exists(config_file): raise IOError("the configuration file %r does not exist\n" % config_file) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # reading the global environ first def _upper(items): return [(key.upper(), value) for key, value in items] global_env = dict(_upper(os.environ.items())) if 'env' in cfg.sections(): global_env.update(dict(_upper(cfg.items('env')))) cfg.set_env(global_env) # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['multicast_endpoint'] = dget('circus', 'multicast_endpoint', DEFAULT_ENDPOINT_MULTICAST) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None) config['statsd'] = dget('circus', 'statsd', False, bool) if config['stats_endpoint'] is None: config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS elif not config['statsd']: warnings.warn( "You defined a stats_endpoint without " "setting up statsd to True.", DeprecationWarning) config['statsd'] = True config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) config['pidfile'] = dget('circus', 'pidfile') config['loglevel'] = dget('circus', 'loglevel') config['logoutput'] = dget('circus', 'logoutput') config['fqdn_prefix'] = dget('circus', 'fqdn_prefix', None, str) # Initialize watchers, plugins & sockets to manage watchers = [] plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugin = dict(cfg.items(section)) plugin['name'] = section plugins.append(plugin) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section, noreplace=True): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timeout': watcher['graceful_timeout'] = dget(section, "graceful_timeout", 30, int) elif opt.startswith('stderr_stream') or \ opt.startswith('stdout_stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) elif opt == 'autostart': watcher['autostart'] = dget(section, "autostart", True, bool) elif opt == 'close_child_stdout': watcher['close_child_stdout'] = dget( section, "close_child_stdout", False, bool) elif opt == 'close_child_stderr': watcher['close_child_stderr'] = dget( section, "close_child_stderr", False, bool) else: # freeform watcher[opt] = val watchers.append(watcher) # Second pass to make sure env sections apply to all watchers. environs = defaultdict(dict) # global env first def _extend(target, source): for name, value in source: if name in target: continue target[name] = value for watcher in watchers: _extend(environs[watcher['name']], global_env.items()) # then per-watcher env for section in cfg.sections(): if section.startswith('env:'): section_elements = section.split("env:", 1)[1] watcher_patterns = [s.strip() for s in section_elements.split(',')] for pattern in watcher_patterns: match = [w for w in watchers if fnmatch(w['name'], pattern)] for watcher in match: watcher_name = watcher['name'] extra = cfg.items(section, noreplace=True) environs[watcher_name].update(_upper(extra)) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() _extend(watcher['env'], environs[watcher['name']].items()) for option, value in watcher.items(): if option in ('name', 'env'): continue if not isinstance(value, str): continue watcher[option] = replace_gnu_args(value, env=watcher['env']) config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config