def spawn_process(self): """Spawn process. """ if self.stopped: return if not self.call_hook("before_spawn"): self.stopped = True return False cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds(), env=self.env) self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None try: process = Process( self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr, ) # stream stderr/stdout if configured if pipe_stdout: self.stdout_redirector.add_redirection("stdout", process, process.stdout) if pipe_stderr: self.stderr_redirector.add_redirection("stderr", process, process.stderr) self.processes[process.pid] = process logger.debug("running %s process [pid %d]", self.name, process.pid) except OSError as e: logger.warning("error in %r: %s", self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return self.stop()
def handle_recv(self, data): """Handle received message from circusd We need to handle two messages: - spawn: add a new monitored child pid - reap: remove a killed child pid from monitoring """ watcher_name, action, msg = self.split_data(data) logger.debug("received data from circusd: watcher.%s.%s, %s", watcher_name, action, msg) # check if monitored watchers: if self._match_watcher_name(watcher_name): try: message = self.load_message(msg) except ValueError: logger.error("Error while decoding json for message: %s", msg) else: if "process_pid" not in message: logger.warning('no process_pid in message') return pid = str(message.get("process_pid")) if action == "spawn": self.pid_status[pid] = dict(watcher=watcher_name, last_activity=time.time()) logger.info("added new monitored pid for %s:%s", watcher_name, pid) # very questionable fix for Py3 here! # had to add check for pid in self.pid_status elif action == "reap" and pid in self.pid_status: old_pid = self.pid_status.pop(pid) logger.info("removed monitored pid for %s:%s", old_pid['watcher'], pid)
def spawn(self): # noinspection PyUnusedLocal socket_names = set(socket_name.lower() for socket_name in self.watcher._get_sockets_fds()) self.cmd = self._fix_socket_name(self.cmd, socket_names) self.args = [self._fix_socket_name(arg, socket_names) for arg in self.args] args = self.format_args() stdout = _bools_to_papa_out(self.pipe_stdout, self.close_child_stdout) stderr = _bools_to_papa_out(self.pipe_stderr, self.close_child_stderr) papa_name = 'circus.{0}.{1}'.format(self.name, self.wid).lower() self._papa_name = papa_name self._papa = papa.Papa() try: p = self._papa.make_process(papa_name, executable=self.executable, args=args, env=self.env, working_dir=self.working_dir, uid=self.uid, gid=self.gid, rlimits=self.rlimits, stdout=stdout, stderr=stderr) except papa.Error: p = self._papa.list_processes(papa_name) if p: p = p[papa_name] logger.warning('Process "%s" wid "%d" already exists in papa. ' 'Using the existing process.', self.name, self.wid) else: raise self._worker = PapaProcessWorker(self, p['pid']) self._papa_watcher = self._papa.watch_processes(papa_name) self.started = p['started']
def spawn_fly(self): """ spawn fly """ if self.stopped: return self._fly_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: fly = None try: fly = Fly(self._fly_counter, self.cmd, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env) self.flies[self._fly_counter] = fly logger.info('running %s fly [pid %d]' % (self.name, fly.pid)) except OSError, e: logger.warning('error in %r: %s' % (self.name, str(e))) if fly is None: nb_tries += 1 continue else: self.send_msg("spawn", {"fly_id": fly.wid, "fly_pid": fly.pid, "time": time.time()}) return
def spawn_process(self): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, env=self.env) nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None try: process = Process(self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if pipe_stdout and self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if pipe_stderr and self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) if not self.call_hook('after_spawn', pid=process.pid): self.kill_process(process) del self.processes[process.pid] return False except OSError as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) return True return False
def spawn_process(self): """Spawn process. """ if self.stopped: return self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: process = None try: process = Process(self._process_counter, self.cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable) self.processes[self._process_counter] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.send_msg("spawn", {"process_id": process.wid, "process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return
def handle_recv(self, data): """Handle received message from circusd We need to handle two messages: - spawn: add a new monitored child pid - reap: remove a killed child pid from monitoring """ topic, msg = data topic_parts = topic.split(".") logger.debug("received data from circusd: %s, %s", topic_parts, msg) # check if monitored watchers: if (topic_parts[0] == 'watcher' and self._match_watcher_name(topic_parts[1])): try: message = json.loads(msg) except ValueError: logger.error("Error while decoding json for message: %s", msg) else: if "process_pid" not in message: logger.warning('no process_pid in message') return pid = str(message.get("process_pid")) if topic_parts[2] == "spawn": self.pid_status[pid] = dict(watcher=topic_parts[1], last_activity=time.time()) logger.info("added new monitored pid for %s:%s", topic_parts[1], pid) elif topic_parts[2] == "reap": old_pid = self.pid_status.pop(pid) logger.info("removed monitored pid for %s:%s", old_pid['watcher'], pid)
def spawn_fly(self): if self.stopped: return self._fly_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: fly = None try: fly = Fly(self._fly_counter, self.cmd, wdir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env) self.flies[self._fly_counter] = fly logger.info('running %s fly [pid %d]' % (self.name, fly.pid)) except OSError, e: logger.warning('error in %r: %s' % (self.name, str(e))) if fly is None: nb_tries += 1 continue else: return
def spawn_process(self): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds(), env=self.env) nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None try: process = Process(self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if pipe_stdout and self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if pipe_stderr and self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) return True return False
def spawn_process(self): """Spawn process. """ if self.stopped: return cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds()) self._process_counter += 1 nb_tries = 0 pipe_stdout = self.stdout_redirector is not None pipe_stderr = self.stderr_redirector is not None while nb_tries < self.max_retry or self.max_retry == -1: process = None try: process = Process(self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if pipe_stdout: self.stdout_redirector.add_redirection( 'stdout', process, process.stdout) if pipe_stderr: self.stderr_redirector.add_redirection( 'stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", { "process_pid": process.pid, "time": time.time() }) time.sleep(self.warmup_delay) return
def spawn_process(self): """Spawn process. """ if self.stopped: return def _repl(matchobj): name = matchobj.group(1) if name in self.sockets: return str(self.sockets[name].fileno()) return '${socket:%s}' % name cmd = re.sub('\$\{socket\:(\w+)\}', _repl, self.cmd) self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: process = None try: process = Process(self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets) # stream stderr/stdout if configured if self.stdout_redirector is not None: self.stdout_redirector.add_redirection( 'stdout', process, process.stdout) if self.stderr_redirector is not None: self.stderr_redirector.add_redirection( 'stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", { "process_pid": process.pid, "time": time.time() }) time.sleep(self.warmup_delay) return
def _init_multicast_endpoint(self): multicast_addr, multicast_port = urlparse(self.multicast_endpoint)\ .netloc.split(':') try: self.udp_socket = create_udp_socket(multicast_addr, multicast_port) self.loop.add_handler(self.udp_socket.fileno(), self.handle_autodiscover_message, ioloop.IOLoop.READ) except (IOError, OSError, ValueError): message = ("Multicast discovery is disabled, there was an " "error during udp socket creation.") logger.warning(message, exc_info=True)
def spawn_process(self): """Spawn process. """ if self.stopped: return def _repl(matchobj): name = matchobj.group(1) if name in self.sockets: return str(self.sockets[name].fileno()) return "${socket:%s}" % name cmd = re.sub("\$\{socket\:(\w+)\}", _repl, self.cmd) self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: process = None try: process = Process( self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, ) # stream stderr/stdout if configured if self.stdout_redirector is not None: self.stdout_redirector.add_redirection("stdout", process, process.stdout) if self.stderr_redirector is not None: self.stderr_redirector.add_redirection("stderr", process, process.stderr) self.processes[process.pid] = process logger.debug("running %s process [pid %d]", self.name, process.pid) except OSError, e: logger.warning("error in %r: %s", self.name, str(e)) if process is None: nb_tries += 1 continue else: self.send_msg("spawn", {"process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return
def _fix_socket_name(self, s, socket_names): if s: s_lower = s.lower() while '$(circus.sockets.' in s_lower: start = s_lower.index('$(circus.sockets.') end = s_lower.index(')', start) socket_name = s_lower[start + 17:end] if socket_name not in socket_names: logger.warning('Process "{0}" refers to socket "{1}" but ' 'they do not have the same "use_papa" state' .format(self.name, socket_name)) s = ''.join((s[:start], '$(socket.circus.', socket_name, '.fileno)', s[end + 1:])) s_lower = s.lower() return s
def receive_udp_socket(self, fd, events): """Check the socket for received UDP message. This method is periodically called by the ioloop. If messages are received and parsed, update the status of the corresponing pid. """ data, _ = self.sock.recvfrom(1024) heartbeat = self._decode_received_udp_message(data) if "pid" in heartbeat: if heartbeat["pid"] in self.pid_status: # TODO: check and compare received time # with our own time.time() self.pid_status[heartbeat["pid"]]["last_activity"] = time.time() else: logger.warning("received watchdog for a" "non monitored process:%s", heartbeat) logger.debug("watchdog message: %s", heartbeat)
def handle_message(self, raw_msg): try: # Handle garbage messages, # which are not originating from circus cid, msg = raw_msg except (TypeError, ValueError): logger.warning("got unexpected message %s", raw_msg) return msg = msg.strip() if not msg: self.send_response(None, cid, msg, "error: empty command") else: logger.debug("got message %s", msg) self.dispatch((cid, msg))
def spawn(self): # noinspection PyUnusedLocal socket_names = set(socket_name.lower() for socket_name in self.watcher._get_sockets_fds()) self.cmd = self._fix_socket_name(self.cmd, socket_names) if isinstance(self.args, string_types): self.args = self._fix_socket_name(self.args, socket_names) else: self.args = [ self._fix_socket_name(arg, socket_names) for arg in self.args ] args = self.format_args() stdout = _bools_to_papa_out(self.pipe_stdout, self.close_child_stdout) stderr = _bools_to_papa_out(self.pipe_stderr, self.close_child_stderr) papa_name = 'circus.{0}.{1}'.format(self.name, self.wid).lower() self._papa_name = papa_name self._papa = papa.Papa() if stderr is None and stdout is None: p = self._papa.list_processes(papa_name) if p: p = p[papa_name] if not p['running']: self._papa.remove_processes(papa_name) try: p = self._papa.make_process(papa_name, executable=self.executable, args=args, env=self.env, working_dir=self.working_dir, uid=self.uid, gid=self.gid, rlimits=self.rlimits, stdout=stdout, stderr=stderr) except papa.Error: p = self._papa.list_processes(papa_name) if p: p = p[papa_name] logger.warning( 'Process "%s" wid "%d" already exists in papa. ' 'Using the existing process.', self.name, self.wid) else: raise self._worker = PapaProcessWorker(self, p['pid']) self._papa_watcher = self._papa.watch_processes(papa_name) self.started = p['started']
def spawn_process(self): """Spawn process. """ if self.stopped: return cmd = util.replace_gnu_args(self.cmd, sockets=self._get_sockets_fds()) self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry or self.max_retry == -1: process = None try: process = Process(self._process_counter, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, _exec=self._exec) # stream stderr/stdout if configured if self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return
def spawn_process(self): """Spawn process. """ if self.stopped: return self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: process = None try: process = Process( self._process_counter, self.cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, ) # stream stderr/stdout if configured if self.stdout_redirector is not None: self.stdout_redirector.add_redirection("stdout", process, process.stdout) if self.stderr_redirector is not None: self.stderr_redirector.add_redirection("stderr", process, process.stderr) self.processes[self._process_counter] = process self.pids[process.pid] = process.wid logger.debug("running %s process [pid %d]", self.name, process.pid) except OSError, e: logger.warning("error in %r: %s", self.name, str(e)) if process is None: nb_tries += 1 continue else: self.send_msg("spawn", {"process_id": process.wid, "process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return
def spawn_process(self): """Spawn process. """ if self.stopped: return self._process_counter += 1 nb_tries = 0 while nb_tries < self.max_retry: process = None try: process = Process(self._process_counter, self.cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable) # stream stderr/stdout if configured if self.stdout_redirector is not None: self.stdout_redirector.add_redirection('stdout', process, process.stdout) if self.stderr_redirector is not None: self.stderr_redirector.add_redirection('stderr', process, process.stderr) self.processes[self._process_counter] = process self.pids[process.pid] = process.wid logger.debug('running %s process [pid %d]', self.name, process.pid) except OSError, e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.send_msg("spawn", {"process_id": process.wid, "process_pid": process.pid, "time": time.time()}) time.sleep(self.warmup_delay) return
def receive_udp_socket(self, fd, events): """Check the socket for received UDP message. This method is periodically called by the ioloop. If messages are received and parsed, update the status of the corresponing pid. """ data, _ = self.sock.recvfrom(1024) heartbeat = self._decode_received_udp_message(data) if "pid" in heartbeat: if heartbeat['pid'] in self.pid_status: # TODO: check and compare received time # with our own time.time() self.pid_status[ heartbeat["pid"]]['last_activity'] = time.time() else: logger.warning( "received watchdog for a" "non monitored process:%s", heartbeat) logger.debug("watchdog message: %s", heartbeat)
def spawn_process(self, recovery_wid=None): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not recovery_wid and not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, env=self.env) nb_tries = 0 # start the redirector now so we can catch any startup errors if self.stream_redirector: self.stream_redirector.start() while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_stream is not None pipe_stderr = self.stderr_stream is not None # noinspection PyPep8Naming ProcCls = self._process_class try: process = ProcCls(self.name, recovery_wid or self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdin=self.close_child_stdin, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if self.stream_redirector: self.stream_redirector.add_redirections(process) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) if not self.call_hook('after_spawn', pid=process.pid): self.kill_process(process) del self.processes[process.pid] return False # catch ValueError as well, as a misconfigured rlimit setting could # lead to bad infinite retries here except (OSError, ValueError) as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", { "process_pid": process.pid, "time": process.started }) return process.started return False
def get_config(config_file): if not os.path.exists(config_file): raise IOError("the configuration file %r does not exist\n" % config_file) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # reading the global environ first def _upper(items): return [(key.upper(), value) for key, value in items] global_env = dict(_upper(os.environ.items())) if 'env' in cfg.sections(): global_env.update(dict(_upper(cfg.items('env')))) cfg.set_env(global_env) # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['multicast_endpoint'] = dget('circus', 'multicast_endpoint', DEFAULT_ENDPOINT_MULTICAST) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None) config['statsd'] = dget('circus', 'statsd', False, bool) if config['stats_endpoint'] is None: config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS elif not config['statsd']: warnings.warn( "You defined a stats_endpoint without " "setting up statsd to True.", DeprecationWarning) config['statsd'] = True config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) config['pidfile'] = dget('circus', 'pidfile') config['loglevel'] = dget('circus', 'loglevel') config['logoutput'] = dget('circus', 'logoutput') config['fqdn_prefix'] = dget('circus', 'fqdn_prefix', None, str) # Initialize watchers, plugins & sockets to manage watchers = [] plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugin = dict(cfg.items(section)) plugin['name'] = section plugins.append(plugin) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section, noreplace=True): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timeout': watcher['graceful_timeout'] = dget(section, "graceful_timeout", 30, int) elif opt.startswith('stderr_stream') or \ opt.startswith('stdout_stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) elif opt == 'autostart': watcher['autostart'] = dget(section, "autostart", True, bool) elif opt == 'close_child_stdout': watcher['close_child_stdout'] = dget( section, "close_child_stdout", False, bool) elif opt == 'close_child_stderr': watcher['close_child_stderr'] = dget( section, "close_child_stderr", False, bool) else: # freeform watcher[opt] = val watchers.append(watcher) # Second pass to make sure env sections apply to all watchers. environs = defaultdict(dict) # global env first def _extend(target, source): for name, value in source: if name in target: continue target[name] = value for watcher in watchers: _extend(environs[watcher['name']], global_env.items()) # then per-watcher env for section in cfg.sections(): if section.startswith('env:'): section_elements = section.split("env:", 1)[1] watcher_patterns = [s.strip() for s in section_elements.split(',')] for pattern in watcher_patterns: match = [w for w in watchers if fnmatch(w['name'], pattern)] for watcher in match: watcher_name = watcher['name'] extra = cfg.items(section, noreplace=True) environs[watcher_name].update(_upper(extra)) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() _extend(watcher['env'], environs[watcher['name']].items()) for option, value in watcher.items(): if option in ('name', 'env'): continue if not isinstance(value, str): continue watcher[option] = replace_gnu_args(value, env=watcher['env']) config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config
def load_from_config(cls, config_file, loop=None): cfg = get_config(config_file) watchers = [] for watcher in cfg.get('watchers', []): try: _watcher = Watcher.load_from_config(watcher) except Exception as e: logger.warning('Failed to create watcher', exc_info=e) else: watchers.append(_watcher) sockets = [] for socket_ in cfg.get('sockets', []): try: _socket = CircusSocket.load_from_config(socket_) except Exception as e: logger.warning('Failed to create socket', exc_info=e) else: sockets.append(_socket) httpd = cfg.get('httpd', False) if httpd: # controlling that we have what it takes to run the web UI # if something is missing this will tell the user try: import circusweb # NOQA except ImportError: logger.error('You need to install circus-web') sys.exit(1) # creating arbiter arbiter = cls(watchers, cfg['endpoint'], cfg['pubsub_endpoint'], check_delay=cfg.get('check_delay', 1.), prereload_fn=cfg.get('prereload_fn'), statsd=cfg.get('statsd', False), stats_endpoint=cfg.get('stats_endpoint'), papa_endpoint=cfg.get('papa_endpoint'), multicast_endpoint=cfg.get('multicast_endpoint'), plugins=cfg.get('plugins'), sockets=sockets, warmup_delay=cfg.get('warmup_delay', 0), httpd=httpd, loop=loop, httpd_host=cfg.get('httpd_host', 'localhost'), httpd_port=cfg.get('httpd_port', 8080), debug=cfg.get('debug', False), debug_gc=cfg.get('debug_gc', False), ssh_server=cfg.get('ssh_server', None), pidfile=cfg.get('pidfile', None), loglevel=cfg.get('loglevel', None), logoutput=cfg.get('logoutput', None), loggerconfig=cfg.get('loggerconfig', None), fqdn_prefix=cfg.get('fqdn_prefix', None), umask=cfg['umask'], endpoint_owner=cfg.get('endpoint_owner', None)) # store the cfg which will be used, so it can be used later # for checking if the cfg has been changed arbiter._cfg = cls.get_arbiter_config(cfg) arbiter.config_file = config_file return arbiter
def __init__(self, name='', host=None, port=None, family=None, type=None, proto=None, backlog=None, path=None, umask=None, replace=None, interface=None, so_reuseport=False): if path is not None: if not hasattr(socket, 'AF_UNIX'): raise NotImplementedError("AF_UNIX not supported on this" " platform") else: family = socket.AF_UNIX host = port = None log_differences = False with papa.Papa() as p: prefixed_name = 'circus.' + name.lower() try: papa_socket = p.make_socket(prefixed_name, host, port, family, type, backlog, path, umask, interface, so_reuseport) except papa.Error: papa_socket = p.list_sockets(prefixed_name) if papa_socket: papa_socket = papa_socket[prefixed_name] log_differences = True else: raise self.name = name self.host = papa_socket.get('host') self.port = papa_socket.get('port') self.family = papa_socket['family'] self.socktype = papa_socket['type'] self.backlog = papa_socket.get('backlog') self.path = papa_socket.get('path') self.umask = papa_socket.get('umask') self.interface = papa_socket.get('interface') self.so_reuseport = papa_socket.get('so_reuseport', False) self._fileno = papa_socket.get('fileno') self.use_papa = True if log_differences: differences = [] if host != self.host: differences.append('host={0}'.format(self.host)) if port != self.port: differences.append('port={0}'.format(self.port)) if backlog != self.backlog: differences.append('backlog={0}'.format(self.backlog)) if path != self.path: differences.append('path={0}'.format(self.path)) if umask != self.umask: differences.append('umask={0}'.format(self.umask)) if interface != self.interface: differences.append('interface={0}'.format(self.interface)) if so_reuseport != self.so_reuseport: differences.append('so_reuseport={0}'.format( self.so_reuseport)) if differences: logger.warning( 'Socket "%s" already exists in papa with ' '%s. Using the existing socket.', name, ' '.join(differences)) self.replace = True
def __init__(self, name, cmd, args=None, numprocesses=1, warmup_delay=0., working_dir=None, shell=False, shell_args=None, uid=None, max_retry=5, gid=None, send_hup=False, stop_signal=signal.SIGTERM, stop_children=False, env=None, graceful_timeout=30.0, prereload_fn=None, rlimits=None, executable=None, stdout_stream=None, stderr_stream=None, priority=0, loop=None, singleton=False, use_sockets=False, copy_env=False, copy_path=False, max_age=0, max_age_variance=30, hooks=None, respawn=True, autostart=True, on_demand=False, virtualenv=None, stdin_socket=None, close_child_stdin=True, close_child_stdout=False, close_child_stderr=False, virtualenv_py_ver=None, use_papa=False, **options): self.name = name self.use_sockets = use_sockets self.on_demand = on_demand self.res_name = name.lower().replace(" ", "_") self.numprocesses = int(numprocesses) self.warmup_delay = warmup_delay self.cmd = cmd self.args = args self._status = "stopped" self.graceful_timeout = float(graceful_timeout) self.prereload_fn = prereload_fn self.executable = None self.priority = priority self.stdout_stream_conf = copy.copy(stdout_stream) self.stderr_stream_conf = copy.copy(stderr_stream) self.stdout_stream = get_stream(self.stdout_stream_conf) self.stderr_stream = get_stream(self.stderr_stream_conf) self.stream_redirector = None self.max_retry = int(max_retry) self._options = options self.singleton = singleton self.copy_env = copy_env self.copy_path = copy_path self.virtualenv = virtualenv self.virtualenv_py_ver = virtualenv_py_ver self.max_age = int(max_age) self.max_age_variance = int(max_age_variance) self.ignore_hook_failure = [ 'before_stop', 'after_stop', 'before_signal', 'after_signal', 'extended_stats' ] self.respawn = respawn self.autostart = autostart self.stdin_socket = stdin_socket self.close_child_stdin = close_child_stdin self.close_child_stdout = close_child_stdout self.close_child_stderr = close_child_stderr self.use_papa = use_papa and papa is not None self.loop = loop or ioloop.IOLoop.current() if singleton and self.numprocesses not in (0, 1): raise ValueError("Cannot have %d processes with a singleton " " watcher" % self.numprocesses) if not singleton and self.numprocesses == 0: logger.warning( "Non-singleton watcher %s with numprocesses=0 " "won't start any process", self.name) if IS_WINDOWS: if self.stdout_stream or self.stderr_stream: raise NotImplementedError("Streams are not supported" " on Windows.") if not copy_env and not env: # Copy the env by default on Windows as we can't run any # executable without some env variables # Eventually, we could set only some required variables, # such as SystemRoot self.copy_env = True self.optnames = ( ("numprocesses", "warmup_delay", "working_dir", "uid", "gid", "send_hup", "stop_signal", "stop_children", "shell", "shell_args", "env", "max_retry", "cmd", "args", "respawn", "graceful_timeout", "executable", "use_sockets", "priority", "copy_env", "singleton", "stdout_stream_conf", "on_demand", "stderr_stream_conf", "max_age", "max_age_variance", "close_child_stdin", "close_child_stdout", "close_child_stderr", "use_papa") + tuple(options.keys())) if not working_dir: # working dir hasn't been set working_dir = util.get_working_dir() self.working_dir = working_dir self.processes = {} self.shell = shell self.shell_args = shell_args self.uid = uid self.gid = gid if self.copy_env: self.env = os.environ.copy() if self.copy_path: path = os.pathsep.join(sys.path) self.env['PYTHONPATH'] = path if env is not None: self.env.update(env) else: if self.copy_path: raise ValueError(('copy_env and copy_path must have the ' 'same value')) self.env = env if self.virtualenv: util.load_virtualenv(self, py_ver=virtualenv_py_ver) # load directories in PYTHONPATH if provided # so if a hook is there, it can be loaded if self.env is not None and 'PYTHONPATH' in self.env: for path in self.env['PYTHONPATH'].split(os.pathsep): if path in sys.path: continue site.addsitedir(path) self.rlimits = rlimits self.send_hup = send_hup self.stop_signal = stop_signal self.stop_children = stop_children self.sockets = self.evpub_socket = None self.arbiter = None self.hooks = {} self._resolve_hooks(hooks) self._found_wids = [] if self.use_papa: with papa.Papa() as p: base_name = 'circus.{0}.*'.format(name.lower()) running = p.list_processes(base_name) self._found_wids = [ int(proc_name[len(base_name) - 1:]) for proc_name in running ]
def spawn_process(self, recovery_wid=None): """Spawn process. Return True if ok, False if the watcher must be stopped """ if self.is_stopped(): return True if not recovery_wid and not self.call_hook('before_spawn'): return False cmd = util.replace_gnu_args(self.cmd, env=self.env) nb_tries = 0 # start the redirector now so we can catch any startup errors if self.stream_redirector: self.stream_redirector.start() while nb_tries < self.max_retry or self.max_retry == -1: process = None pipe_stdout = self.stdout_stream is not None pipe_stderr = self.stderr_stream is not None # noinspection PyPep8Naming ProcCls = self._process_class try: process = ProcCls(self.name, recovery_wid or self._nextwid, cmd, args=self.args, working_dir=self.working_dir, shell=self.shell, uid=self.uid, gid=self.gid, env=self.env, rlimits=self.rlimits, executable=self.executable, use_fds=self.use_sockets, watcher=self, pipe_stdout=pipe_stdout, pipe_stderr=pipe_stderr, close_child_stdin=self.close_child_stdin, close_child_stdout=self.close_child_stdout, close_child_stderr=self.close_child_stderr) # stream stderr/stdout if configured if self.stream_redirector: self.stream_redirector.add_redirections(process) self.processes[process.pid] = process logger.debug('running %s process [pid %d]', self.name, process.pid) if not self.call_hook('after_spawn', pid=process.pid): self.kill_process(process) del self.processes[process.pid] return False # catch ValueError as well, as a misconfigured rlimit setting could # lead to bad infinite retries here except (OSError, ValueError) as e: logger.warning('error in %r: %s', self.name, str(e)) if process is None: nb_tries += 1 continue else: self.notify_event("spawn", {"process_pid": process.pid, "time": process.started}) return process.started return False
def __init__(self, name='', host=None, port=None, family=None, type=None, proto=None, backlog=None, path=None, umask=None, replace=None, interface=None, so_reuseport=False): if path is not None: if not hasattr(socket, 'AF_UNIX'): raise NotImplementedError("AF_UNIX not supported on this" " platform") else: family = socket.AF_UNIX host = port = None log_differences = False with papa.Papa() as p: prefixed_name = 'circus.' + name.lower() try: papa_socket = p.make_socket(prefixed_name, host, port, family, type, backlog, path, umask, interface, so_reuseport) except papa.Error: papa_socket = p.list_sockets(prefixed_name) if papa_socket: papa_socket = papa_socket[prefixed_name] log_differences = True else: raise self.name = name self.host = papa_socket.get('host') self.port = papa_socket.get('port') self.family = papa_socket['family'] self.socktype = papa_socket['type'] self.backlog = papa_socket.get('backlog') self.path = papa_socket.get('path') self.umask = papa_socket.get('umask') self.interface = papa_socket.get('interface') self.so_reuseport = papa_socket.get('so_reuseport', False) self._fileno = papa_socket.get('fileno') self.use_papa = True if log_differences: differences = [] if host != self.host: differences.append('host={0}'.format(self.host)) if port != self.port: differences.append('port={0}'.format(self.port)) if backlog != self.backlog: differences.append('backlog={0}'.format(self.backlog)) if path != self.path: differences.append('path={0}'.format(self.path)) if umask != self.umask: differences.append('umask={0}'.format(self.umask)) if interface != self.interface: differences.append('interface={0}'.format(self.interface)) if so_reuseport != self.so_reuseport: differences.append('so_reuseport={0}'.format( self.so_reuseport)) if differences: logger.warning('Socket "%s" already exists in papa with ' '%s. Using the existing socket.', name, ' '.join(differences)) self.replace = True
def __init__( self, name="", host=None, port=None, family=None, type=None, proto=None, backlog=None, path=None, umask=None, replace=None, interface=None, so_reuseport=False, blocking=False, ): if path is not None: if not hasattr(socket, "AF_UNIX"): raise NotImplementedError("AF_UNIX not supported on this" " platform") else: family = socket.AF_UNIX host = port = None log_differences = False with papa.Papa() as p: prefixed_name = "circus." + name.lower() try: papa_socket = p.make_socket( prefixed_name, host, port, family, type, backlog, path, umask, interface, so_reuseport ) except papa.Error: papa_socket = p.list_sockets(prefixed_name) if papa_socket: papa_socket = papa_socket[prefixed_name] log_differences = True else: raise self.name = name self.host = papa_socket.get("host") self.port = papa_socket.get("port") self.family = papa_socket["family"] self.socktype = papa_socket["type"] self.backlog = papa_socket.get("backlog") self.path = papa_socket.get("path") self.umask = papa_socket.get("umask") self.interface = papa_socket.get("interface") self.so_reuseport = papa_socket.get("so_reuseport", False) self._fileno = papa_socket.get("fileno") self.use_papa = True if log_differences: differences = [] if host != self.host: differences.append("host={0}".format(self.host)) if port != self.port: differences.append("port={0}".format(self.port)) if backlog != self.backlog: differences.append("backlog={0}".format(self.backlog)) if path != self.path: differences.append("path={0}".format(self.path)) if umask != self.umask: differences.append("umask={0}".format(self.umask)) if interface != self.interface: differences.append("interface={0}".format(self.interface)) if so_reuseport != self.so_reuseport: differences.append("so_reuseport={0}".format(self.so_reuseport)) if differences: logger.warning( 'Socket "%s" already exists in papa with ' "%s. Using the existing socket.", name, " ".join(differences), ) self.replace = True
def get_config(config_file): if not os.path.exists(config_file): sys.stderr.write("the configuration file %r does not exist\n" % config_file) sys.stderr.write("Exiting...\n") sys.exit(1) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str) config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) stream_backend = dget('circus', 'stream_backend', 'thread') if stream_backend == 'gevent': try: import gevent # NOQA except ImportError: sys.stderr.write("stream_backend set to gevent, " + "but gevent isn't installed\n") sys.stderr.write("Exiting...\n") sys.exit(1) from gevent import monkey monkey.patch_all() try: import zmq.green as zmq # NOQA except ImportError: try: from gevent_zeromq import monkey_patch except ImportError: sys.stderr.write("stream_backend set to gevent, but " + "but required PyZMQ >= 2.2.0.1 not found\n") sys.stderr.write("Exiting...\n") sys.exit(1) monkey_patch() warnings.warn("gevent_zeromq is deprecated, please " "use PyZMQ >= 2.2.0.1") config['stream_backend'] = stream_backend # Initialize watchers, plugins & sockets to manage watchers = [] environs = {} plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugins.append(dict(cfg.items(section))) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timout': watcher['graceful_timeout'] = dget( section, "graceful_timeout", 30, int) elif opt.startswith('stderr_stream') or \ opt.startswith('stdout_stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'stream_backend': watcher['stream_backend'] = val elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) else: # freeform watcher[opt] = val # set the stream backend if 'stream_backend' not in watcher: watcher['stream_backend'] = stream_backend watchers.append(watcher) if section.startswith('env:'): for watcher in section.split("env:", 1)[1].split(','): watcher = watcher.strip() if not watcher in environs: environs[watcher] = dict() environs[watcher].update([(k.upper(), v) for k, v in cfg.items(section)]) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() watcher['env'].update(environs[watcher['name']]) config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config
def get_config(config_file): if not os.path.exists(config_file): raise IOError("the configuration file %r does not exist\n" % config_file) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['multicast_endpoint'] = dget('circus', 'multicast_endpoint', DEFAULT_ENDPOINT_MULTICAST) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None) config['statsd'] = dget('circus', 'statsd', False, bool) if config['stats_endpoint'] is None: config['stats_endpoint'] = DEFAULT_ENDPOINT_STATS elif not config['statsd']: warnings.warn("You defined a stats_endpoint without " "setting up statsd to True.", DeprecationWarning) config['statsd'] = True config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) config['pidfile'] = dget('circus', 'pidfile') config['loglevel'] = dget('circus', 'loglevel') config['logoutput'] = dget('circus', 'logoutput') # Initialize watchers, plugins & sockets to manage watchers = [] environs = {} plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugins.append(dict(cfg.items(section))) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timeout': watcher['graceful_timeout'] = dget( section, "graceful_timeout", 30, int) elif opt.startswith('stderr_stream') or \ opt.startswith('stdout_stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) elif opt == 'autostart': watcher['autostart'] = dget(section, "autostart", True, bool) elif opt == 'close_child_stdout': watcher['close_child_stdout'] = dget(section, "close_child_stdout", False, bool) elif opt == 'close_child_stderr': watcher['close_child_stderr'] = dget(section, "close_child_stderr", False, bool) else: # freeform watcher[opt] = val watchers.append(watcher) # Second pass to make sure env sections apply to all watchers. for section in cfg.sections(): if section.startswith('env:'): section_elements = section.split("env:", 1)[1] watcher_patterns = [s.strip() for s in section_elements.split(',')] for pattern in watcher_patterns: matching_watchers = [ w for w in watchers if fnmatch(w['name'], pattern) ] for watcher in matching_watchers: watcher_name = watcher['name'] if not watcher_name in environs: environs[watcher_name] = dict() environs[watcher_name].update( [(k.upper(), v) for k, v in cfg.items(section)]) if section == 'env': for watcher in watchers: environs[watcher['name']].update( [(k.upper(), v) for k, v in cfg.items(section)]) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() watcher['env'].update(environs[watcher['name']]) config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config
def get_config(config_file): if not os.path.exists(config_file): sys.stderr.write("the configuration file %r does not exist\n" % config_file) sys.stderr.write("Exiting...\n") sys.exit(1) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str) config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) # Initialize watchers, plugins & sockets to manage watchers = [] environs = {} plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugins.append(dict(cfg.items(section))) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timeout': watcher['graceful_timeout'] = dget(section, "graceful_timeout", 30, int) elif opt.startswith('stderr_stream') or \ opt.startswith('stdout_stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) elif opt == 'autostart': watcher['autostart'] = dget(section, "autostart", True, bool) else: # freeform watcher[opt] = val watchers.append(watcher) if section.startswith('env:'): for watcher in section.split("env:", 1)[1].split(','): watcher = watcher.strip() if not watcher in environs: environs[watcher] = dict() environs[watcher].update([(k.upper(), v) for k, v in cfg.items(section)]) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() watcher['env'].update(environs[watcher['name']]) config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config
def get_config(config_file): if not os.path.exists(config_file): sys.stderr.write("the configuration file %r does not exist\n" % config_file) sys.stderr.write("Exiting...\n") sys.exit(1) cfg, cfg_files_read = read_config(config_file) dget = cfg.dget config = {} # main circus options config['check'] = dget('circus', 'check_delay', 5, int) config['endpoint'] = dget('circus', 'endpoint', DEFAULT_ENDPOINT_DEALER) config['pubsub_endpoint'] = dget('circus', 'pubsub_endpoint', DEFAULT_ENDPOINT_SUB) config['stats_endpoint'] = dget('circus', 'stats_endpoint', None, str) config['warmup_delay'] = dget('circus', 'warmup_delay', 0, int) config['httpd'] = dget('circus', 'httpd', False, bool) config['httpd_host'] = dget('circus', 'httpd_host', 'localhost', str) config['httpd_port'] = dget('circus', 'httpd_port', 8080, int) config['debug'] = dget('circus', 'debug', False, bool) # Initialize watchers, plugins & sockets to manage watchers = [] environs = {} plugins = [] sockets = [] for section in cfg.sections(): if section.startswith("socket:"): sock = dict(cfg.items(section)) sock['name'] = section.split("socket:")[-1].lower() sockets.append(sock) if section.startswith("plugin:"): plugins.append(dict(cfg.items(section))) if section.startswith("watcher:"): watcher = watcher_defaults() watcher['name'] = section.split("watcher:", 1)[1] # create watcher options for opt, val in cfg.items(section): if opt == 'cmd': watcher['cmd'] = val elif opt == 'args': watcher['args'] = val elif opt == 'numprocesses': watcher['numprocesses'] = dget(section, 'numprocesses', 1, int) elif opt == 'warmup_delay': watcher['warmup_delay'] = dget(section, 'warmup_delay', 0, int) elif opt == 'executable': watcher['executable'] = dget(section, 'executable', None, str) elif opt == 'working_dir': watcher['working_dir'] = val elif opt == 'shell': watcher['shell'] = dget(section, 'shell', False, bool) elif opt == 'uid': watcher['uid'] = val elif opt == 'gid': watcher['gid'] = val elif opt == 'send_hup': watcher['send_hup'] = dget(section, 'send_hup', False, bool) elif opt == 'check_flapping': watcher['check_flapping'] = dget(section, 'check_flapping', True, bool) elif opt == 'max_retry': watcher['max_retry'] = dget(section, "max_retry", 5, int) elif opt == 'graceful_timout': watcher['graceful_timeout'] = dget( section, "graceful_timeout", 30, int) elif opt.startswith('publish'): X, publish_opt = opt.split(".", 1) watcher['publish'][publish_opt] = val elif opt.startswith('stream'): stream_name, stream_opt = opt.split(".", 1) watcher[stream_name][stream_opt] = val elif opt.startswith('rlimit_'): limit = opt[7:] watcher['rlimits'][limit] = int(val) elif opt == 'priority': watcher['priority'] = dget(section, "priority", 0, int) elif opt == 'use_sockets': watcher['use_sockets'] = dget(section, "use_sockets", False, bool) elif opt == 'singleton': watcher['singleton'] = dget(section, "singleton", False, bool) elif opt == 'copy_env': watcher['copy_env'] = dget(section, "copy_env", False, bool) elif opt == 'copy_path': watcher['copy_path'] = dget(section, "copy_path", False, bool) elif opt.startswith('hooks.'): hook_name = opt[len('hooks.'):] val = [elmt.strip() for elmt in val.split(',', 1)] if len(val) == 1: val.append(False) else: val[1] = to_boolean(val[1]) watcher['hooks'][hook_name] = val elif opt == 'respawn': watcher['respawn'] = dget(section, "respawn", True, bool) elif opt == 'env': logger.warning('the env option is deprecated the use of ' 'env sections is recommended') watcher['env'] = parse_env_str(val) elif opt == 'autostart': watcher['autostart'] = dget(section, "autostart", True, bool) else: # freeform watcher[opt] = val watchers.append(watcher) if section.startswith('env:'): for watcher in section.split("env:", 1)[1].split(','): watcher = watcher.strip() if not watcher in environs: environs[watcher] = dict() environs[watcher].update([(k.upper(), v) for k, v in cfg.items(section)]) for watcher in watchers: if watcher['name'] in environs: if not 'env' in watcher: watcher['env'] = dict() watcher['env'].update(environs[watcher['name']]) plugin_streams = ['stream_stdout', 'stream_stderr'] for stream in plugin_streams: if stream in watcher: plugin = create_plugin_from_stream(watcher[stream], watcher['name'], stream) if plugin: plugins.append(plugin) publishers = watcher.get('publish', {}) publishers[stream.split('_')[1]] = True watcher['publish'] = publishers config['watchers'] = watchers config['plugins'] = plugins config['sockets'] = sockets return config