def _start_child(self, wrap): if len(wrap.forktimes) > wrap.workers: # Limit ourselves to one process a second (over the period of # number of workers * 1 second). This will allow workers to # start up quickly but ensure we don't fork off children that # die instantly too quickly. if time.time() - wrap.forktimes[0] < wrap.workers: LOG.info(_LI('Forking too fast, sleeping')) time.sleep(1) wrap.forktimes.pop(0) wrap.forktimes.append(time.time()) pid = os.fork() if pid == 0: self.launcher = self._child_process(wrap.service) while True: self._child_process_handle_signal() status, signo = self._child_wait_for_exit_or_signal( self.launcher) if not _is_sighup_and_daemon(signo): self.launcher.wait() break self.launcher.restart() os._exit(status) LOG.info(_LI('Started child %d'), pid) wrap.children.add(pid) self.children[pid] = wrap return pid
def wait(self): """Loop waiting on children to die and respawning as necessary.""" systemd.notify_once() if self.conf.log_options: LOG.debug('Full set of CONF:') self.conf.log_opt_values(LOG, logging.DEBUG) try: while True: self.handle_signal() self._respawn_children() # No signal means that stop was called. Don't clean up here. if not self.sigcaught: return signame = _signals_to_name[self.sigcaught] LOG.info(_LI('Caught %s, stopping children'), signame) if not _is_sighup_and_daemon(self.sigcaught): break self.conf.reload_config_files() for service in set( [wrap.service for wrap in self.children.values()]): service.reset() for pid in self.children: os.kill(pid, signal.SIGHUP) self.running = True self.sigcaught = None except eventlet.greenlet.GreenletExit: LOG.info(_LI("Wait called after thread killed. Cleaning up.")) self.stop()
def wait(self): """Loop waiting on children to die and respawning as necessary.""" systemd.notify_once() LOG.debug('Full set of CONF:') CONF.log_opt_values(LOG, logging.DEBUG) try: while True: self.handle_signal() self._respawn_children() # No signal means that stop was called. Don't clean up here. if not self.sigcaught: return signame = _signo_to_signame(self.sigcaught) LOG.info(_LI('Caught %s, stopping children'), signame) if not _is_sighup_and_daemon(self.sigcaught): break cfg.CONF.reload_config_files() for service in set( [wrap.service for wrap in self.children.values()]): service.reset() for pid in self.children: os.kill(pid, signal.SIGHUP) self.running = True self.sigcaught = None except eventlet.greenlet.GreenletExit: LOG.info(_LI("Wait called after thread killed. Cleaning up.")) self.stop()
def _start_child(self, wrap): if len(wrap.forktimes) > wrap.workers: # Limit ourselves to one process a second (over the period of # number of workers * 1 second). This will allow workers to # start up quickly but ensure we don't fork off children that # die instantly too quickly. if time.time() - wrap.forktimes[0] < wrap.workers: LOG.info(_LI('Forking too fast, sleeping')) time.sleep(1) wrap.forktimes.pop(0) wrap.forktimes.append(time.time()) pid = os.fork() if pid == 0: launcher = self._child_process(wrap.service) while True: self._child_process_handle_signal() status, signo = self._child_wait_for_exit_or_signal(launcher) if not _is_sighup_and_daemon(signo): break launcher.restart() os._exit(status) LOG.info(_LI('Started child %d'), pid) wrap.children.add(pid) self.children[pid] = wrap return pid
def _wait_child(self): try: # Don't block if no child processes have exited pid, status = os.waitpid(0, os.WNOHANG) if not pid: return None except OSError as exc: if exc.errno not in (errno.EINTR, errno.ECHILD): raise return None if os.WIFSIGNALED(status): sig = os.WTERMSIG(status) LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'), dict(pid=pid, sig=sig)) else: code = os.WEXITSTATUS(status) LOG.info(_LI('Child %(pid)s exited with status %(code)d'), dict(pid=pid, code=code)) if pid not in self.children: LOG.warning(_LW('pid %d not in child list'), pid) return None wrap = self.children.pop(pid) wrap.children.remove(pid) return wrap
def _add_periodic_task(cls, task): """Add a periodic task to the list of periodic tasks. The task should already be decorated by @periodic_task. :return: whether task was actually enabled """ name = task._periodic_name if task._periodic_spacing < 0: LOG.info( _LI('Skipping periodic task %(task)s because ' 'its interval is negative'), {'task': name}) return False if not task._periodic_enabled: LOG.info( _LI('Skipping periodic task %(task)s because ' 'it is disabled'), {'task': name}) return False # A periodic spacing of zero indicates that this task should # be run on the default interval to avoid running too # frequently. if task._periodic_spacing == 0: task._periodic_spacing = DEFAULT_INTERVAL cls._periodic_tasks.append((name, task)) cls._periodic_spacing[name] = task._periodic_spacing return True
def _add_periodic_task(cls, task): """Add a periodic task to the list of periodic tasks. The task should already be decorated by @periodic_task. :return: whether task was actually enabled """ name = task._periodic_name if task._periodic_spacing < 0: LOG.info(_LI('Skipping periodic task %(task)s because ' 'its interval is negative'), {'task': name}) return False if not task._periodic_enabled: LOG.info(_LI('Skipping periodic task %(task)s because ' 'it is disabled'), {'task': name}) return False # A periodic spacing of zero indicates that this task should # be run on the default interval to avoid running too # frequently. if task._periodic_spacing == 0: task._periodic_spacing = DEFAULT_INTERVAL cls._periodic_tasks.append((name, task)) cls._periodic_spacing[name] = task._periodic_spacing return True
def _get_socket(self, host, port, backlog): bind_addr = (host, port) # TODO(dims): eventlet's green dns/socket module does not actually # support IPv6 in getaddrinfo(). We need to get around this in the # future or monitor upstream for a fix try: info = socket.getaddrinfo(bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)[0] family = info[0] bind_addr = info[-1] except Exception: family = socket.AF_INET try: sock = eventlet.listen(bind_addr, family, backlog=backlog) except EnvironmentError: LOG.error(_LE("Could not bind to %(host)s:%(port)s"), {'host': host, 'port': port}) raise sock = self._set_socket_opts(sock) LOG.info(_LI("%(name)s listening on %(host)s:%(port)s"), {'name': self.name, 'host': host, 'port': port}) return sock
def initialize_if_enabled(conf): conf.register_opts(_options.eventlet_backdoor_opts) backdoor_locals = { "exit": _dont_use_this, # So we don't exit the entire process "quit": _dont_use_this, # So we don't exit the entire process "fo": _find_objects, "pgt": _print_greenthreads, "pnt": _print_nativethreads, } if conf.backdoor_port is None: return None start_port, end_port = _parse_port_range(str(conf.backdoor_port)) # NOTE(johannes): The standard sys.displayhook will print the value of # the last expression and set it to __builtin__._, which overwrites # the __builtin__._ that gettext sets. Let's switch to using pprint # since it won't interact poorly with gettext, and it's easier to # read the output too. def displayhook(val): if val is not None: pprint.pprint(val) sys.displayhook = displayhook sock = _listen("localhost", start_port, end_port, eventlet.listen) # In the case of backdoor port being zero, a port number is assigned by # listen(). In any case, pull the port number out here. port = sock.getsockname()[1] LOG.info(_LI("Eventlet backdoor listening on %(port)s for process %(pid)d"), {"port": port, "pid": os.getpid()}) eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, locals=backdoor_locals) return port
def _get_socket(self, host, port, backlog): bind_addr = (host, port) # TODO(dims): eventlet's green dns/socket module does not actually # support IPv6 in getaddrinfo(). We need to get around this in the # future or monitor upstream for a fix try: info = socket.getaddrinfo(bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)[0] family = info[0] bind_addr = info[-1] except Exception: family = socket.AF_INET try: sock = eventlet.listen(bind_addr, family, backlog=backlog) except EnvironmentError: LOG.error(_LE("Could not bind to %(host)s:%(port)s"), { 'host': host, 'port': port }) raise sock = self._set_socket_opts(sock) LOG.info(_LI("%(name)s listening on %(host)s:%(port)s"), { 'name': self.name, 'host': host, 'port': port }) return sock
def _get_unix_socket(self, socket_file, socket_mode, backlog): sock = eventlet.listen(socket_file, family=socket.AF_UNIX, backlog=backlog) if socket_mode is not None: os.chmod(socket_file, socket_mode) LOG.info(_LI("%(name)s listening on %(socket_file)s:"), {'name': self.name, 'socket_file': socket_file}) return sock
def _pipe_watcher(self): # This will block until the write end is closed when the parent # dies unexpectedly self.readpipe.read(1) LOG.info(_LI('Parent process has died unexpectedly, exiting')) sys.exit(1)
def wait(self): """Loop waiting on children to die and respawning as necessary.""" systemd.notify_once() if self.conf.log_options: LOG.debug('Full set of CONF:') self.conf.log_opt_values(LOG, logging.DEBUG) try: while True: self.handle_signal() self._respawn_children() # No signal means that stop was called. Don't clean up here. if not self.sigcaught: return signame = self.signal_handler.signals_to_name[self.sigcaught] LOG.info(_LI('Caught %s, stopping children'), signame) if not _is_sighup_and_daemon(self.sigcaught): break if self.restart_method == 'reload': self.conf.reload_config_files() elif self.restart_method == 'mutate': self.conf.mutate_config_files() for service in set( [wrap.service for wrap in self.children.values()]): service.reset() for pid in self.children: os.kill(pid, signal.SIGTERM) self.running = True self.sigcaught = None except eventlet.greenlet.GreenletExit: LOG.info(_LI("Wait called after thread killed. Cleaning up.")) # if we are here it means that we are trying to do graceful shutdown. # add alarm watching that graceful_shutdown_timeout is not exceeded if (self.conf.graceful_shutdown_timeout and self.signal_handler.is_signal_supported('SIGALRM')): signal.alarm(self.conf.graceful_shutdown_timeout) self.stop()
def _pipe_watcher(self): # This will block until the write end is closed when the parent # dies unexpectedly self.readpipe.read(1) LOG.info(_LI('Parent process has died unexpectedly, exiting')) if self.launcher: self.launcher.stop() sys.exit(1)
def stop(self): """Stops eventlet server. Doesn't allow accept new connecting. :returns: None """ LOG.info(_LI("Stopping WSGI server.")) if self._server is not None: # let eventlet close socket self._pool.resize(0) self._server.kill()
def launch_service(self, service, workers=1): """Launch a service with a given number of workers. :param service: a service to launch, must be an instance of :class:`oslo_service.service.ServiceBase` :param workers: a number of processes in which a service will be running """ _check_service_base(service) wrap = ServiceWrapper(service, workers) LOG.info(_LI('Starting %d workers'), wrap.workers) while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap)
def wait(self): """Block, until the server has stopped. Waits on the server's eventlet to finish, then returns. :returns: None """ try: if self._server is not None: num = self._pool.running() LOG.debug("Waiting WSGI server to finish %d requests.", num) self._pool.waitall() except greenlet.GreenletExit: LOG.info(_LI("WSGI server has stopped."))
def stop(self): """Terminate child processes and wait on each.""" self.running = False for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_LI('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def _initialize_if_enabled(conf): conf.register_opts(_options.eventlet_backdoor_opts) backdoor_locals = { 'exit': _dont_use_this, # So we don't exit the entire process 'quit': _dont_use_this, # So we don't exit the entire process 'fo': _find_objects, 'pgt': _print_greenthreads, 'pnt': _print_nativethreads, } if conf.backdoor_port is None and conf.backdoor_socket is None: return None if conf.backdoor_socket is None: start_port, end_port = _parse_port_range(str(conf.backdoor_port)) sock = _listen('localhost', start_port, end_port, eventlet.listen) # In the case of backdoor port being zero, a port number is assigned by # listen(). In any case, pull the port number out here. where_running = sock.getsockname()[1] else: sock = _try_open_unix_domain_socket(conf.backdoor_socket) where_running = conf.backdoor_socket # NOTE(johannes): The standard sys.displayhook will print the value of # the last expression and set it to __builtin__._, which overwrites # the __builtin__._ that gettext sets. Let's switch to using pprint # since it won't interact poorly with gettext, and it's easier to # read the output too. def displayhook(val): if val is not None: pprint.pprint(val) sys.displayhook = displayhook LOG.info( _LI('Eventlet backdoor listening on %(where_running)s for' ' process %(pid)d'), { 'where_running': where_running, 'pid': os.getpid() }) thread = eventlet.spawn(eventlet.backdoor.backdoor_server, sock, locals=backdoor_locals) return (where_running, thread)
def _wait_for_exit_or_signal(self): status = None signo = 0 if self.conf.log_options: LOG.debug("Full set of CONF:") self.conf.log_opt_values(LOG, logging.DEBUG) try: super(ServiceLauncher, self).wait() except SignalExit as exc: signame = self.signal_handler.signals_to_name[exc.signo] LOG.info(_LI("Caught %s, exiting"), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code except Exception: self.stop() return status, signo
def _initialize_if_enabled(conf): conf.register_opts(_options.eventlet_backdoor_opts) backdoor_locals = { 'exit': _dont_use_this, # So we don't exit the entire process 'quit': _dont_use_this, # So we don't exit the entire process 'fo': _find_objects, 'pgt': _print_greenthreads, 'pnt': _print_nativethreads, } if conf.backdoor_port is None and conf.backdoor_socket is None: return None if conf.backdoor_socket is None: start_port, end_port = _parse_port_range(str(conf.backdoor_port)) sock = _listen('localhost', start_port, end_port, eventlet.listen) # In the case of backdoor port being zero, a port number is assigned by # listen(). In any case, pull the port number out here. where_running = sock.getsockname()[1] else: sock = _try_open_unix_domain_socket(conf.backdoor_socket) where_running = conf.backdoor_socket # NOTE(johannes): The standard sys.displayhook will print the value of # the last expression and set it to __builtin__._, which overwrites # the __builtin__._ that gettext sets. Let's switch to using pprint # since it won't interact poorly with gettext, and it's easier to # read the output too. def displayhook(val): if val is not None: pprint.pprint(val) sys.displayhook = displayhook LOG.info( _LI('Eventlet backdoor listening on %(where_running)s for' ' process %(pid)d'), {'where_running': where_running, 'pid': os.getpid()} ) thread = eventlet.spawn(eventlet.backdoor.backdoor_server, sock, locals=backdoor_locals) return (where_running, thread)
def stop(self): """Terminate child processes and wait on each.""" self.running = False LOG.debug("Stop services.") for service in set([wrap.service for wrap in self.children.values()]): service.stop() LOG.debug("Killing children.") for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_LI("Waiting on %d children to exit"), len(self.children)) while self.children: self._wait_child()
def _child_wait_for_exit_or_signal(self, launcher): status = 0 signo = 0 # NOTE(johannes): All exceptions are caught to ensure this # doesn't fallback into the loop spawning children. It would # be bad for a child to spawn more children. try: launcher.wait() except SignalExit as exc: signame = self.signal_handler.signals_to_name[exc.signo] LOG.info(_LI('Child caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code except BaseException: LOG.exception(_LE('Unhandled exception')) status = 2 return status, signo
def stop(self): """Terminate child processes and wait on each.""" self.running = False LOG.debug("Stop services.") for service in set([wrap.service for wrap in self.children.values()]): service.stop() LOG.debug("Killing children.") for pid in self.children: try: os.kill(pid, signal.SIGTERM) except OSError as exc: if exc.errno != errno.ESRCH: raise # Wait for children to die if self.children: LOG.info(_LI('Waiting on %d children to exit'), len(self.children)) while self.children: self._wait_child()
def _wait_for_exit_or_signal(self): status = None signo = 0 if self.conf.log_options: LOG.debug('Full set of CONF:') self.conf.log_opt_values(LOG, logging.DEBUG) try: super(ServiceLauncher, self).wait() except SignalExit as exc: signame = self.signal_handler.signals_to_name[exc.signo] LOG.info(_LI('Caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: self.stop() status = exc.code except Exception: self.stop() return status, signo
def _wait_for_exit_or_signal(self, ready_callback=None): status = None signo = 0 LOG.debug('Full set of CONF:') CONF.log_opt_values(LOG, logging.DEBUG) try: if ready_callback: ready_callback() super(ServiceLauncher, self).wait() except SignalExit as exc: signame = _signo_to_signame(exc.signo) LOG.info(_LI('Caught %s, exiting'), signame) status = exc.code signo = exc.signo except SystemExit as exc: status = exc.code finally: self.stop() return status, signo
def __init__(self, conf, name, app, host='0.0.0.0', port=0, pool_size=None, protocol=eventlet.wsgi.HttpProtocol, backlog=128, use_ssl=False, max_url_len=None): """Initialize, but do not start, a WSGI server. :param conf: Instance of ConfigOpts. :param name: Pretty name for logging. :param app: The WSGI application to serve. :param host: IP address to serve the application. :param port: Port number to server the application. :param pool_size: Maximum number of eventlets to spawn concurrently. :param protocol: Protocol class. :param backlog: Maximum number of queued connections. :param use_ssl: Wraps the socket in an SSL context if True. :param max_url_len: Maximum length of permitted URLs. :returns: None :raises: InvalidInput :raises: EnvironmentError """ self.conf = conf self.conf.register_opts(_options.wsgi_opts) self.default_pool_size = self.conf.wsgi_default_pool_size # Allow operators to customize http requests max header line size. eventlet.wsgi.MAX_HEADER_LINE = conf.max_header_line self.name = name self.app = app self._server = None self._protocol = protocol self.pool_size = pool_size or self.default_pool_size self._pool = eventlet.GreenPool(self.pool_size) self._logger = logging.getLogger("eventlet.wsgi.server") self._use_ssl = use_ssl self._max_url_len = max_url_len self.client_socket_timeout = conf.client_socket_timeout or None if backlog < 1: raise InvalidInput(reason=_('The backlog must be more than 0')) bind_addr = (host, port) # TODO(dims): eventlet's green dns/socket module does not actually # support IPv6 in getaddrinfo(). We need to get around this in the # future or monitor upstream for a fix try: info = socket.getaddrinfo(bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)[0] family = info[0] bind_addr = info[-1] except Exception: family = socket.AF_INET if self._use_ssl: sslutils.is_enabled(conf) try: self._socket = eventlet.listen(bind_addr, family, backlog=backlog) except EnvironmentError: LOG.error(_LE("Could not bind to %(host)s:%(port)s"), {'host': host, 'port': port}) raise (self.host, self.port) = self._socket.getsockname()[0:2] LOG.info(_LI("%(name)s listening on %(host)s:%(port)s"), {'name': self.name, 'host': self.host, 'port': self.port})
def _fast_exit(self, *args): LOG.info(_LI("Caught SIGINT signal, instantaneous exiting")) os._exit(1)
def __init__(self, conf, name, app, host='0.0.0.0', port=0, pool_size=None, protocol=eventlet.wsgi.HttpProtocol, backlog=128, use_ssl=False, max_url_len=None): """Initialize, but do not start, a WSGI server. :param name: Pretty name for logging. :param app: The WSGI application to serve. :param host: IP address to serve the application. :param port: Port number to server the application. :param pool_size: Maximum number of eventlets to spawn concurrently. :param backlog: Maximum number of queued connections. :param max_url_len: Maximum length of permitted URLs. :returns: None :raises: InvalidInput """ self.conf = conf self.conf.register_opts(_options.wsgi_opts) self.default_pool_size = self.conf.wsgi_default_pool_size # Allow operators to customize http requests max header line size. eventlet.wsgi.MAX_HEADER_LINE = conf.max_header_line self.name = name self.app = app self._server = None self._protocol = protocol self.pool_size = pool_size or self.default_pool_size self._pool = eventlet.GreenPool(self.pool_size) self._logger = logging.getLogger("eventlet.wsgi.server") self._use_ssl = use_ssl self._max_url_len = max_url_len self.client_socket_timeout = conf.client_socket_timeout or None self.default_pool_size = conf.wsgi_default_pool_size if backlog < 1: raise InvalidInput(reason='The backlog must be more than 0') bind_addr = (host, port) # TODO(dims): eventlet's green dns/socket module does not actually # support IPv6 in getaddrinfo(). We need to get around this in the # future or monitor upstream for a fix try: info = socket.getaddrinfo(bind_addr[0], bind_addr[1], socket.AF_UNSPEC, socket.SOCK_STREAM)[0] family = info[0] bind_addr = info[-1] except Exception: family = socket.AF_INET if self._use_ssl: sslutils.is_enabled(conf) try: self._socket = eventlet.listen(bind_addr, family, backlog=backlog) except EnvironmentError: LOG.error(_LE("Could not bind to %(host)s:%(port)s"), { 'host': host, 'port': port }) raise (self.host, self.port) = self._socket.getsockname()[0:2] LOG.info(_LI("%(name)s listening on %(host)s:%(port)s"), { 'name': self.name, 'host': self.host, 'port': self.port })
def _on_alarm_exit(self, signo, frame): LOG.info(_LI('Graceful shutdown timeout exceeded, ' 'instantaneous exiting')) os._exit(1)
def _on_timeout_exit(self, *args): LOG.info(_LI('Graceful shutdown timeout exceeded, ' 'instantaneous exiting')) os._exit(1)
def _on_timeout_exit(self, *args): LOG.info( _LI('Graceful shutdown timeout exceeded, ' 'instantaneous exiting')) os._exit(1)
def launch_service(self, service, workers=1): wrap = ServiceWrapper(service, workers) LOG.info(_LI('Starting %d workers'), wrap.workers) while self.running and len(wrap.children) < wrap.workers: self._start_child(wrap)
def _on_alarm_exit(self, signo, frame): LOG.info( _LI('Graceful shutdown timeout exceeded, ' 'instantaneous exiting')) os._exit(1)
def _fast_exit(self, signo, frame): LOG.info(_LI('Caught SIGINT signal, instantaneous exiting')) os._exit(1)