def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.pid = "[booting]" self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.aborted = False self.reloader = None self.nr = 0 if cfg.max_requests > 0: jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter else: self.max_requests = sys.maxsize self.alive = True self.log = log self.tmp = WorkerTmp(cfg)
def __init__(self, usage=None, prog=None): self.usage = usage self.cfg = None self.callable = None self.prog = prog self.logger = None self.do_load_config() self.tmp = WorkerTmp(self.cfg)
def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 # number of requests self.max_requests = cfg.max_requests or MAXSIZE self.alive = True self.log = log self.tmp = WorkerTmp(cfg) # instrumentation self.use_statsd = cfg.statsd_host is not None if self.use_statsd: self.log.info("Worker will send stats to {0}".format(cfg.statsd_host)) self.last_nr = 0 # store nr at the last instrumentation call self.last_usr_t = 0 # store last user time from os.times() self.statsd = statsd(cfg.statsd_host, self.log)
def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or MAXSIZE self.alive = True self.log = log self.tmp = WorkerTmp(cfg)
def __init__(self, age, ppid, socket, app, timeout, cfg): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.socket = socket self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or sys.maxint self.alive = True self.log = logging.getLogger(__name__) self.debug = cfg.debug self.address = self.socket.getsockname() self.tmp = WorkerTmp(cfg)
def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.pid = "[booting]" self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg # 是否已启动 self.booted = False # master.murder_workers 中会修改 aborted, 但是好像其他地方没有使用 self.aborted = False # worker是否alive, 当not alive时, (一般)run中的main loop会退出 self.alive = True # num of requests self.nr = 0 self.reloader = None # The maximum number of requests a worker will process before restarting # This is a simple method to help limit the damage of memory leaks. if cfg.max_requests > 0: jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter else: self.max_requests = sys.maxsize self.log = log logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) self.logger = logger # 用于worker的health check self.tmp = WorkerTmp(cfg)
def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.pid = "[booting]" self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.aborted = False self.reloader = None self.nr = 0 jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter or sys.maxsize self.alive = True self.log = log self.tmp = WorkerTmp(cfg)
class Worker(object): SIGNALS = map( lambda x: getattr(signal, "SIG%s" % x), "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() ) PIPE = [] def __init__(self, age, ppid, socket, app, timeout, cfg): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.socket = socket self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or sys.maxint self.alive = True self.log = logging.getLogger(__name__) self.debug = cfg.debug self.address = self.socket.getsockname() self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() map(util.set_non_blocking, self.PIPE) map(util.close_on_exec, self.PIPE) # Prevent fd inherientence util.close_on_exec(self.socket) util.close_on_exec(self.tmp.fileno()) self.init_signals() self.wsgi = self.app.wsgi() # Enter main run loop self.booted = True self.run() def init_signals(self): map(lambda s: signal.signal(s, signal.SIG_DFL), self.SIGNALS) signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) signal.signal(signal.SIGWINCH, self.handle_winch) def handle_quit(self, sig, frame): self.alive = False def handle_exit(self, sig, frame): self.alive = False sys.exit(0) def handle_error(self, client, exc): self.log.exception("Error handling request") status_int = 500 reason = "Internal Server Error" mesg = "" if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName,)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "<p>Invalid Request Line '%s'</p>" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "<p>Invalid Method '%s'</p>" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "<p>Invalid HTTP Version '%s'</p>" % str(exc) elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)): mesg = "<p>Invalid Header '%s'</p>" % str(exc) if self.debug: tb = traceback.format_exc() mesg += "<h2>Traceback:</h2>\n<pre>%s</pre>" % tb try: util.write_error(client, status_int, reason, mesg) except: self.log.warning("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class Worker(object): SIGNALS = map( lambda x: getattr(signal, "SIG%s" % x), "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() ) PIPE = [] def __init__(self, age, ppid, socket, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.socket = socket self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or sys.maxint self.alive = True self.log = log self.debug = cfg.debug self.address = self.socket.getsockname() self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() map(util.set_non_blocking, self.PIPE) map(util.close_on_exec, self.PIPE) # Prevent fd inherientence util.close_on_exec(self.socket) util.close_on_exec(self.tmp.fileno()) self.log.close_on_exec() self.init_signals() self.wsgi = self.app.wsgi() # Enter main run loop self.booted = True self.run() def init_signals(self): map(lambda s: signal.signal(s, signal.SIG_DFL), self.SIGNALS) signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) # Don't let SIGQUIT and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGQUIT, False) signal.siginterrupt(signal.SIGUSR1, False) def handle_usr1(self, sig, frame): self.log.reopen_files() def handle_quit(self, sig, frame): self.alive = False def handle_exit(self, sig, frame): self.alive = False sys.exit(0) def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders,)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "<p>Invalid Request Line '%s'</p>" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "<p>Invalid Method '%s'</p>" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "<p>Invalid HTTP Version '%s'</p>" % str(exc) elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)): mesg = "<p>Invalid Header '%s'</p>" % str(exc) elif isinstance(exc, LimitRequestLine): mesg = "<p>%s</p>" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "<p>Error parsing headers: '%s'</p>" % str(exc) self.log.debug("Invalid request from ip={ip}: {error}"\ "".format(ip=addr[0], error=str(exc), ) ) else: self.log.exception("Error handling request") status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) if self.debug: tb = traceback.format_exc() mesg += "<h2>Traceback:</h2>\n<pre>%s</pre>" % tb try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class Worker(object): SIGNALS = [getattr(signal, "SIG%s" % x) \ for x in "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] PIPE = [] def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or MAXSIZE self.alive = True self.log = log self.debug = cfg.debug self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ # set enviroment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() for p in self.PIPE: util.set_non_blocking(p) util.close_on_exec(p) # Prevent fd inherientence [util.close_on_exec(s) for s in self.sockets] util.close_on_exec(self.tmp.fileno()) self.log.close_on_exec() self.init_signals() self.wsgi = self.app.wsgi() self.cfg.post_worker_init(self) # Enter main run loop self.booted = True self.run() def init_signals(self): # reset signaling [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) # Don't let SIGQUIT and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGQUIT, False) signal.siginterrupt(signal.SIGUSR1, False) def handle_usr1(self, sig, frame): self.log.reopen_files() def handle_quit(self, sig, frame): self.alive = False def handle_exit(self, sig, frame): # Log stack trace of all threads just before terminating. # Useful for tracking down long requests that may have stalled # for any number of reasons which may include incompatible gevent libraries # Snippet is taken from: # http://stackoverflow.com/questions/132058/showing-the-stack-trace-from-a-running-python-application import threading, sys, traceback id2name = dict([(th.ident, th.name) for th in threading.enumerate()]) code = [] for threadId, stack in sys._current_frames().items(): code.append("\n# Thread: %s(%d)" % (id2name.get(threadId,""), threadId)) for filename, lineno, name, line in traceback.extract_stack(stack): code.append('File: "%s", line %d, in %s' % (filename, lineno, name)) if line: code.append(" %s" % (line.strip())) self.log.critical("\n".join(code)) self.alive = False sys.exit(0) def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders, InvalidProxyLine, ForbiddenProxyRequest,)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "<p>Invalid Request Line '%s'</p>" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "<p>Invalid Method '%s'</p>" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "<p>Invalid HTTP Version '%s'</p>" % str(exc) elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)): mesg = "<p>%s</p>" % str(exc) if not req and hasattr(exc, "req"): req = exc.req # for access log elif isinstance(exc, LimitRequestLine): mesg = "<p>%s</p>" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "<p>Error parsing headers: '%s'</p>" % str(exc) elif isinstance(exc, InvalidProxyLine): mesg = "<p>'%s'</p>" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): reason = "Forbidden" mesg = "<p>Request forbidden</p>" status_int = 403 self.log.debug("Invalid request from ip={ip}: {error}"\ "".format(ip=addr[0], error=str(exc), ) ) else: self.log.exception("Error handling request") status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class Worker(object): SIGNALS = [getattr(signal, "SIG%s" % x) \ for x in "ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] PIPE = [] def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.aborted = False self.nr = 0 jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter or MAXSIZE self.alive = True self.log = log self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ # start the reloader if self.cfg.reload: def changed(fname): self.log.info("Worker reloading: %s modified", fname) os.kill(self.pid, signal.SIGQUIT) raise SystemExit() Reloader(callback=changed).start() # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() for p in self.PIPE: util.set_non_blocking(p) util.close_on_exec(p) # Prevent fd inheritance [util.close_on_exec(s) for s in self.sockets] util.close_on_exec(self.tmp.fileno()) self.log.close_on_exec() self.init_signals() self.wsgi = self.app.wsgi() self.cfg.post_worker_init(self) # Enter main run loop self.booted = True self.run() def init_signals(self): # reset signaling [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_quit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) signal.signal(signal.SIGABRT, self.handle_abort) # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGTERM, False) signal.siginterrupt(signal.SIGUSR1, False) def handle_usr1(self, sig, frame): self.log.reopen_files() def handle_exit(self, sig, frame): self.alive = False def handle_quit(self, sig, frame): self.alive = False # worker_int callback self.cfg.worker_int(self) sys.exit(0) def handle_abort(self, sig, frame): self.alive = False self.cfg.worker_abort(self) sys.exit(1) def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders, InvalidProxyLine, ForbiddenProxyRequest,)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "Invalid Request Line '%s'" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "Invalid Method '%s'" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "Invalid HTTP Version '%s'" % str(exc) elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)): mesg = "%s" % str(exc) if not req and hasattr(exc, "req"): req = exc.req # for access log elif isinstance(exc, LimitRequestLine): mesg = "%s" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "Error parsing headers: '%s'" % str(exc) elif isinstance(exc, InvalidProxyLine): mesg = "'%s'" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): reason = "Forbidden" mesg = "Request forbidden" status_int = 403 self.log.debug("Invalid request from ip={ip}: {error}"\ "".format(ip=addr[0], error=str(exc), ) ) else: self.log.exception("Error handling request") status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client, self.cfg) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class BaseApplication(object): """ An application interface for configuring and loading the various necessities for any given web framework. """ def __init__(self, usage=None, prog=None): self.usage = usage self.cfg = None self.callable = None self.prog = prog self.logger = None self.do_load_config() self.tmp = WorkerTmp(self.cfg) def do_load_config(self): """ Loads the configuration """ try: self.load_default_config() self.load_config() except Exception as e: print("\nError: %s" % str(e), file=sys.stderr) sys.stderr.flush() sys.exit(1) def load_default_config(self): # init configuration self.cfg = Config(self.usage, prog=self.prog) def init(self, parser, opts, args): raise NotImplementedError def load(self): raise NotImplementedError def load_config(self): """ This method is used to load the configuration from one or several input(s). Custom Command line, configuration file. You have to override this method in your class. """ raise NotImplementedError def reload(self): self.do_load_config() if self.cfg.spew: debug.spew() def wsgi(self): if self.callable is None: self.callable = self.load() self.add_prometheus_endpoint() return self.callable def run(self): try: Arbiter(self).run() except RuntimeError as e: print("\nError: %s\n" % e, file=sys.stderr) sys.stderr.flush() sys.exit(1) def add_prometheus_endpoint(self): if not self.cfg.prometheus_path: return original_callable = self.callable def callable_with_prometheus(environ, start_response): if environ['REQUEST_METHOD'] == 'GET' and \ environ['PATH_INFO'] == self.cfg.prometheus_path: return self.handle_prometheus_request(environ, start_response) return original_callable(environ, start_response) self.callable = callable_with_prometheus def handle_prometheus_request(self, environ, start_response): start_response('200 OK', [('Content-Type', 'text/plain')]) return pickle.loads(self.tmp.read()).prometheus_dump(),
class Worker(object): SIGNALS = [getattr(signal, "SIG%s" % x) for x in "ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] PIPE = [] def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.pid = "[booting]" self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.aborted = False self.reloader = None self.nr = 0 jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter or MAXSIZE self.alive = True self.log = log self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v util.set_owner_process(self.cfg.uid, self.cfg.gid, initgroups=self.cfg.initgroups) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() for p in self.PIPE: util.set_non_blocking(p) util.close_on_exec(p) # Prevent fd inheritance for s in self.sockets: util.close_on_exec(s) util.close_on_exec(self.tmp.fileno()) self.wait_fds = self.sockets + [self.PIPE[0]] self.log.close_on_exec() self.init_signals() # start the reloader if self.cfg.reload: def changed(fname): self.log.info("Worker reloading: %s modified", fname) self.alive = False self.cfg.worker_int(self) time.sleep(0.1) sys.exit(0) reloader_cls = reloader_engines[self.cfg.reload_engine] self.reloader = reloader_cls(extra_files=self.cfg.reload_extra_files, callback=changed) self.reloader.start() self.load_wsgi() self.cfg.post_worker_init(self) # Enter main run loop self.booted = True self.run() def load_wsgi(self): try: self.wsgi = self.app.wsgi() except SyntaxError as e: if not self.cfg.reload: raise self.log.exception(e) # fix from PR #1228 # storing the traceback into exc_tb will create a circular reference. # per https://docs.python.org/2/library/sys.html#sys.exc_info warning, # delete the traceback after use. try: _, exc_val, exc_tb = sys.exc_info() self.reloader.add_extra_file(exc_val.filename) tb_string = six.StringIO() traceback.print_tb(exc_tb, file=tb_string) self.wsgi = util.make_fail_app(tb_string.getvalue()) finally: del exc_tb def init_signals(self): # reset signaling for s in self.SIGNALS: signal.signal(s, signal.SIG_DFL) # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_quit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) signal.signal(signal.SIGABRT, self.handle_abort) # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGTERM, False) signal.siginterrupt(signal.SIGUSR1, False) if hasattr(signal, 'set_wakeup_fd'): signal.set_wakeup_fd(self.PIPE[1]) def handle_usr1(self, sig, frame): self.log.reopen_files() def handle_exit(self, sig, frame): self.alive = False def handle_quit(self, sig, frame): self.alive = False # worker_int callback self.cfg.worker_int(self) time.sleep(0.1) sys.exit(0) def handle_abort(self, sig, frame): self.alive = False self.cfg.worker_abort(self) sys.exit(1) def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders, InvalidProxyLine, ForbiddenProxyRequest, InvalidSchemeHeaders, SSLError)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "Invalid Request Line '%s'" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "Invalid Method '%s'" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "Invalid HTTP Version '%s'" % str(exc) elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)): mesg = "%s" % str(exc) if not req and hasattr(exc, "req"): req = exc.req # for access log elif isinstance(exc, LimitRequestLine): mesg = "%s" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "Error parsing headers: '%s'" % str(exc) elif isinstance(exc, InvalidProxyLine): mesg = "'%s'" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): reason = "Forbidden" mesg = "Request forbidden" status_int = 403 elif isinstance(exc, InvalidSchemeHeaders): mesg = "%s" % str(exc) elif isinstance(exc, SSLError): reason = "Forbidden" mesg = "'%s'" % str(exc) status_int = 403 msg = "Invalid request from ip={ip}: {error}" self.log.debug(msg.format(ip=addr[0], error=str(exc))) else: if hasattr(req, "uri"): self.log.exception("Error handling request %s", req.uri) status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client, self.cfg) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. self.log.debug("worker: SIGWINCH ignored.")
class Worker(object): SIGNALS = map( lambda x: getattr(signal, "SIG%s" % x), "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() ) PIPE = [] def __init__(self, age, ppid, socket, app, timeout, cfg): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.socket = socket self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or sys.maxint self.alive = True self.log = logging.getLogger(__name__) self.debug = cfg.debug self.address = self.socket.getsockname() self.tmp = WorkerTmp() def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator random.seed() # For waking ourselves up self.PIPE = os.pipe() map(util.set_non_blocking, self.PIPE) map(util.close_on_exec, self.PIPE) # Prevent fd inherientence util.close_on_exec(self.socket) util.close_on_exec(self.tmp.fileno()) self.init_signals() self.wsgi = self.app.wsgi() # Enter main run loop self.booted = True self.run() def init_signals(self): map(lambda s: signal.signal(s, signal.SIG_DFL), self.SIGNALS) signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) signal.signal(signal.SIGWINCH, self.handle_winch) def handle_quit(self, sig, frame): self.alive = False def handle_exit(self, sig, frame): self.alive = False sys.exit(0) def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class Worker(object): SIGNALS = [ getattr(signal, "SIG%s" % x) for x in "ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() ] PIPE = [] def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.pid = "[booting]" self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.aborted = False self.reloader = None self.nr = 0 jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter or MAXSIZE self.alive = True self.log = log self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v util.set_owner_process(self.cfg.uid, self.cfg.gid, initgroups=self.cfg.initgroups) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() for p in self.PIPE: util.set_non_blocking(p) util.close_on_exec(p) # Prevent fd inheritance [util.close_on_exec(s) for s in self.sockets] util.close_on_exec(self.tmp.fileno()) self.wait_fds = self.sockets + [self.PIPE[0]] self.log.close_on_exec() self.init_signals() # start the reloader if self.cfg.reload: def changed(fname): self.log.info("Worker reloading: %s modified", fname) self.alive = False self.cfg.worker_int(self) time.sleep(0.1) sys.exit(0) reloader_cls = reloader_engines[self.cfg.reload_engine] self.reloader = reloader_cls( extra_files=self.cfg.reload_extra_files, callback=changed) self.reloader.start() self.load_wsgi() self.cfg.post_worker_init(self) # Enter main run loop self.booted = True self.run() def load_wsgi(self): try: self.wsgi = self.app.wsgi() except SyntaxError as e: if self.cfg.reload == 'off': raise self.log.exception(e) # fix from PR #1228 # storing the traceback into exc_tb will create a circular reference. # per https://docs.python.org/2/library/sys.html#sys.exc_info warning, # delete the traceback after use. try: exc_type, exc_val, exc_tb = sys.exc_info() self.reloader.add_extra_file(exc_val.filename) tb_string = six.StringIO() traceback.print_tb(exc_tb, file=tb_string) self.wsgi = util.make_fail_app(tb_string.getvalue()) finally: del exc_tb def init_signals(self): # reset signaling [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_quit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) signal.signal(signal.SIGABRT, self.handle_abort) # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGTERM, False) signal.siginterrupt(signal.SIGUSR1, False) if hasattr(signal, 'set_wakeup_fd'): signal.set_wakeup_fd(self.PIPE[1]) def handle_usr1(self, sig, frame): self.log.reopen_files() def handle_exit(self, sig, frame): self.alive = False def handle_quit(self, sig, frame): self.alive = False # worker_int callback self.cfg.worker_int(self) time.sleep(0.1) sys.exit(0) def handle_abort(self, sig, frame): self.alive = False self.cfg.worker_abort(self) sys.exit(1) def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders, InvalidProxyLine, ForbiddenProxyRequest, SSLError)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "Invalid Request Line '%s'" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "Invalid Method '%s'" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "Invalid HTTP Version '%s'" % str(exc) elif isinstance(exc, ( InvalidHeaderName, InvalidHeader, )): mesg = "%s" % str(exc) if not req and hasattr(exc, "req"): req = exc.req # for access log elif isinstance(exc, LimitRequestLine): mesg = "%s" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "Error parsing headers: '%s'" % str(exc) elif isinstance(exc, InvalidProxyLine): mesg = "'%s'" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): reason = "Forbidden" mesg = "Request forbidden" status_int = 403 elif isinstance(exc, SSLError): reason = "Forbidden" mesg = "'%s'" % str(exc) status_int = 403 msg = "Invalid request from ip={ip}: {error}" self.log.debug(msg.format(ip=addr[0], error=str(exc))) else: if hasattr(req, "uri"): self.log.exception("Error handling request %s", req.uri) status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client, self.cfg) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. self.log.debug("worker: SIGWINCH ignored.")
class Worker(object): SIGNALS = map( lambda x: getattr(signal, "SIG%s" % x), "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() ) PIPE = [] def __init__(self, age, ppid, socket, app, timeout, cfg): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.socket = socket self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or sys.maxint self.alive = True self.log = logging.getLogger(__name__) self.debug = cfg.debug self.address = self.socket.getsockname() self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator random.seed() # For waking ourselves up self.PIPE = os.pipe() map(util.set_non_blocking, self.PIPE) map(util.close_on_exec, self.PIPE) # Prevent fd inherientence util.close_on_exec(self.socket) util.close_on_exec(self.tmp.fileno()) self.init_signals() self.wsgi = self.app.wsgi() # Enter main run loop self.booted = True self.run() def init_signals(self): map(lambda s: signal.signal(s, signal.SIG_DFL), self.SIGNALS) signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) signal.signal(signal.SIGWINCH, self.handle_winch) def handle_quit(self, sig, frame): self.alive = False def handle_exit(self, sig, frame): self.alive = False sys.exit(0) def handle_error(self, client, exc): self.log.exception("Error hanlding request") status_int = 500 reason = "Internal Server Error" mesg = "" if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName,)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "<p>Invalid Request Line '%s'</p>" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "<p>Invalid Method'%s'</p>" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "<p>Invalid HTTP Version '%s'</p>" % str(exc) elif isinstance(exc, (InvalidHeaderName, InvalidHeader,)): mesg = "<p>Invalid Header'%s'</p>" % str(exc) if self.debug: tb = traceback.format_exc() mesg += "<h2>Traceback:</h2>\n<pre>%s</pre>" % tb try: util.write_error(client, status_int, reason, mesg) except: self.log.exception("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class Worker(object): SIGNALS = [getattr(signal, "SIG%s" % x) \ for x in "HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split()] PIPE = [] def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg self.booted = False self.nr = 0 self.max_requests = cfg.max_requests or MAXSIZE self.alive = True self.log = log self.debug = cfg.debug self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid @property def pid(self): return os.getpid() def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ raise NotImplementedError() def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super(MyWorkerClass, self).init_process() so that the ``run()`` loop is initiated. """ # set enviroment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v util.set_owner_process(self.cfg.uid, self.cfg.gid) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() for p in self.PIPE: util.set_non_blocking(p) util.close_on_exec(p) # Prevent fd inherientence [util.close_on_exec(s) for s in self.sockets] util.close_on_exec(self.tmp.fileno()) self.log.close_on_exec() self.init_signals() self.wsgi = self.app.wsgi() self.cfg.post_worker_init(self) # Enter main run loop self.booted = True self.run() def init_signals(self): # reset signaling [signal.signal(s, signal.SIG_DFL) for s in self.SIGNALS] # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_exit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) # Don't let SIGQUIT and SIGUSR1 disturb active requests # by interrupting system calls if hasattr(signal, 'siginterrupt'): # python >= 2.6 signal.siginterrupt(signal.SIGQUIT, False) signal.siginterrupt(signal.SIGUSR1, False) def handle_usr1(self, sig, frame): self.log.reopen_files() def handle_quit(self, sig, frame): self.alive = False def handle_exit(self, sig, frame): self.alive = False sys.exit(0) def handle_error(self, req, client, addr, exc): request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, ( InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders, InvalidProxyLine, ForbiddenProxyRequest, )): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "<p>Invalid Request Line '%s'</p>" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "<p>Invalid Method '%s'</p>" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "<p>Invalid HTTP Version '%s'</p>" % str(exc) elif isinstance(exc, ( InvalidHeaderName, InvalidHeader, )): mesg = "<p>%s</p>" % str(exc) if not req and hasattr(exc, "req"): req = exc.req # for access log elif isinstance(exc, LimitRequestLine): mesg = "<p>%s</p>" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "<p>Error parsing headers: '%s'</p>" % str(exc) elif isinstance(exc, InvalidProxyLine): mesg = "<p>'%s'</p>" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): reason = "Forbidden" mesg = "<p>Request forbidden</p>" status_int = 403 self.log.debug("Invalid request from ip={ip}: {error}"\ "".format(ip=addr[0], error=str(exc), ) ) else: self.log.exception("Error handling request") status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) if self.debug: tb = traceback.format_exc() mesg += "<h2>Traceback:</h2>\n<pre>%s</pre>" % tb try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. return
class Worker(object): SIGNALS = [ getattr(signal, "SIG%s" % x) for x in "ABRT HUP QUIT INT TERM USR1 USR2 WINCH CHLD".split() ] PIPE = [] def __init__(self, age, ppid, sockets, app, timeout, cfg, log): """\ This is called pre-fork so it shouldn't do anything to the current process. If there's a need to make process wide changes you'll want to do that in ``self.init_process()``. """ self.age = age self.pid = "[booting]" self.ppid = ppid self.sockets = sockets self.app = app self.timeout = timeout self.cfg = cfg # 是否已启动 self.booted = False # master.murder_workers 中会修改 aborted, 但是好像其他地方没有使用 self.aborted = False # worker是否alive, 当not alive时, (一般)run中的main loop会退出 self.alive = True # num of requests self.nr = 0 self.reloader = None # The maximum number of requests a worker will process before restarting # This is a simple method to help limit the damage of memory leaks. if cfg.max_requests > 0: jitter = randint(0, cfg.max_requests_jitter) self.max_requests = cfg.max_requests + jitter else: self.max_requests = sys.maxsize self.log = log logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stdout)) self.logger = logger # 用于worker的health check self.tmp = WorkerTmp(cfg) def __str__(self): return "<Worker %s>" % self.pid # done: notify every timeout seconds 以便master认为worker还活着 def notify(self): """\ Your worker subclass must arrange to have this method called once every ``self.timeout`` seconds. If you fail in accomplishing this task, the master process will murder your workers. """ # 周期性执行 # self._log('notify') self.tmp.notify() def run(self): """\ This is the mainloop of a worker process. You should override this method in a subclass to provide the intended behaviour for your particular evil schemes. """ self._log('run') raise NotImplementedError() # done: PIPE, signals, wsgi, run def init_process(self): """\ If you override this method in a subclass, the last statement in the function should be to call this method with super().init_process() so that the ``run()`` loop is initiated. """ self._log('init_process') # set environment' variables if self.cfg.env: for k, v in self.cfg.env.items(): os.environ[k] = v # 设置进程所属user group util.set_owner_process(self.cfg.uid, self.cfg.gid, self.cfg.initgroups) # Reseed the random number generator util.seed() # For waking ourselves up self.PIPE = os.pipe() for p in self.PIPE: util.set_non_blocking(p) util.close_on_exec(p) # Prevent fd inheritance for s in self.sockets: util.close_on_exec(s) util.close_on_exec(self.tmp.fileno()) self.wait_fds = self.sockets + [self.PIPE[0]] self.log.close_on_exec() self.init_signals() self._start_reloader() self.load_wsgi() self.cfg.post_worker_init(self) # Enter main run loop self.booted = True self.run() # done def _start_reloader(self): # start the reloader if not self.cfg.reload: return def changed(fname): self._log('reload worker fname=%s' % fname) # self.log.info("Worker reloading: %s modified", fname) self.alive = False self.cfg.worker_int(self) time.sleep(0.1) # sys.exit() 相当于 raise SystemExit, 当worker进程收到 # SystemExit 后会退出, 然后master会重新启动一个worker sys.exit(0) reloader_cls = reloader_engines[self.cfg.reload_engine] # reloader主要是判断(监听)是否需要reload, callback是具体的reload方法 self.reloader = reloader_cls(extra_files=self.cfg.reload_extra_files, callback=changed) # reloader 是一个线程 self.reloader.start() # done def load_wsgi(self): self._log('load_wsgi') try: self.wsgi = self.app.wsgi() except SyntaxError as e: if not self.cfg.reload: raise self.log.exception(e) # fix from PR #1228 # store the traceback into exc_tb will create a circular reference. # per https://docs.python.org/2/library/sys.html#sys.exc_info # warning, delete the traceback after use. try: _, exc_val, exc_tb = sys.exc_info() self.reloader.add_extra_file(exc_val.filename) tb_string = io.StringIO() traceback.print_tb(exc_tb, file=tb_string) self.wsgi = util.make_fail_app(tb_string.getvalue()) finally: del exc_tb # done def init_signals(self): self._log('init_signals') # reset signaling for s in self.SIGNALS: signal.signal(s, signal.SIG_DFL) # SIG_DFL: default handler # init new signaling signal.signal(signal.SIGQUIT, self.handle_quit) signal.signal(signal.SIGTERM, self.handle_exit) signal.signal(signal.SIGINT, self.handle_quit) signal.signal(signal.SIGWINCH, self.handle_winch) signal.signal(signal.SIGUSR1, self.handle_usr1) signal.signal(signal.SIGABRT, self.handle_abort) # Don't let SIGTERM and SIGUSR1 disturb active requests # by interrupting system calls # If the flag argument is false then system calls will # be restarted if interrupted by the specified signal. # 当worker收到SIGTERM SIGUSR1时, 不会立马退出时, 而是等到请求 # 处理完以后再退出, 也即暂时忽略了SIGTERM(?) 那么worker又是如何 # 知道啥时该退出了呢?(请求处理完?), 参见 :file:`log_term_worker.log` signal.siginterrupt(signal.SIGTERM, False) signal.siginterrupt(signal.SIGUSR1, False) if hasattr(signal, 'set_wakeup_fd'): signal.set_wakeup_fd(self.PIPE[1]) # todo def handle_usr1(self, sig, frame): self._log('handle_usr1: %s' % sig) self.log.reopen_files() # done def handle_exit(self, sig, frame): self._log('handle_exit: %s' % sig) self.alive = False # todo: 当Ctrl-C的时候, worker会收到sig=SIGINT, 然后master进程 # 在收到SIGINT后会kill_worker, 也会触发该方法, 此时sig=SIGQUIT, # 所以问题是: 该方法会被调用2次!? 参见 :file:`log_ctrl_c.log` def handle_quit(self, sig, frame): self._log('handle_quit: %s sys.exit(0)' % sig) self.alive = False # worker_int callback self.cfg.worker_int(self) time.sleep(0.1) sys.exit(0) # done def handle_abort(self, sig, frame): self._log('handle_abort: %s & sys.exit(1)' % sig) self.alive = False self.cfg.worker_abort(self) sys.exit(1) # done def handle_error(self, req, client, addr, exc): self._log('handle_error %s' % str(exc)) request_start = datetime.now() addr = addr or ('', -1) # unix socket case if isinstance(exc, (InvalidRequestLine, InvalidRequestMethod, InvalidHTTPVersion, InvalidHeader, InvalidHeaderName, LimitRequestLine, LimitRequestHeaders, InvalidProxyLine, ForbiddenProxyRequest, InvalidSchemeHeaders, SSLError)): status_int = 400 reason = "Bad Request" if isinstance(exc, InvalidRequestLine): mesg = "Invalid Request Line '%s'" % str(exc) elif isinstance(exc, InvalidRequestMethod): mesg = "Invalid Method '%s'" % str(exc) elif isinstance(exc, InvalidHTTPVersion): mesg = "Invalid HTTP Version '%s'" % str(exc) elif isinstance(exc, ( InvalidHeaderName, InvalidHeader, )): mesg = "%s" % str(exc) if not req and hasattr(exc, "req"): req = exc.req # for access log elif isinstance(exc, LimitRequestLine): mesg = "%s" % str(exc) elif isinstance(exc, LimitRequestHeaders): mesg = "Error parsing headers: '%s'" % str(exc) elif isinstance(exc, InvalidProxyLine): mesg = "'%s'" % str(exc) elif isinstance(exc, ForbiddenProxyRequest): reason = "Forbidden" mesg = "Request forbidden" status_int = 403 elif isinstance(exc, InvalidSchemeHeaders): mesg = "%s" % str(exc) elif isinstance(exc, SSLError): reason = "Forbidden" mesg = "'%s'" % str(exc) status_int = 403 msg = "Invalid request from ip={ip}: {error}" self.log.debug(msg.format(ip=addr[0], error=str(exc))) else: if hasattr(req, "uri"): self.log.exception("Error handling request %s", req.uri) status_int = 500 reason = "Internal Server Error" mesg = "" if req is not None: request_time = datetime.now() - request_start environ = default_environ(req, client, self.cfg) environ['REMOTE_ADDR'] = addr[0] environ['REMOTE_PORT'] = str(addr[1]) resp = Response(req, client, self.cfg) resp.status = "%s %s" % (status_int, reason) resp.response_length = len(mesg) self.log.access(resp, req, environ, request_time) try: util.write_error(client, status_int, reason, mesg) except: self.log.debug("Failed to send error message.") # done def handle_winch(self, sig, fname): # Ignore SIGWINCH in worker. Fixes a crash on OpenBSD. self._log('handle_winch: %s' % sig) self.log.debug("worker: SIGWINCH ignored.") def _log(self, msg): colors = { 0: (Fore.RED, '---'), 1: (Fore.GREEN, '+++'), 2: (Fore.BLUE, '***'), 3: (Fore.YELLOW, '==='), 4: (Fore.CYAN, '>>>'), } color = colors[self.pid % 5] self.log.info( '%s %s {%s} %s' % (color[0], color[1], threading.current_thread().ident, msg))