def init_signals(self):
"""Initialize master signal handling. Most of the signals
are queued. Child signals only wake up the master.
"""
self._log('init_signals')
# close old PIPE
for p in self.PIPE:
os.close(p)
# initialize the pipe
# 当收到信号后, 会在 wakeup 中向 PIPE 中写入数据
# 在 sleep 中会通过 select 监听 PIPE 中是否有数据
# todo: 那么问题是? 为啥要搞个PIPE?
self.PIPE = pair = os.pipe()
for p in pair:
util.set_non_blocking(p)
util.close_on_exec(p)
self.log.close_on_exec()
# initialize all signals
for s in self.SIGNALS:
signal.signal(s, self.signal)
# 子进程终止时发送给父进程的信号
signal.signal(signal.SIGCHLD, self.handle_chld)
def __init__(self, workerid, ppid, socket, app, timeout):
self.id = workerid
self.ppid = ppid
# 为什么要除？？
self.timeout = timeout / 2.0
# 这个临时文件用来做什么？
fd, tmpname = tempfile.mkstemp()
self.tmp = os.fdopen(fd, "r+b")
self.tmpname = tmpname
# prevent inherientence
# ？？
self.socket = socket
util.close_on_exec(self.socket)
# 非阻塞
self.socket.setblocking(0)
util.close_on_exec(fd)
self.address = self.socket.getsockname()
self.app = app
self.alive = True
self.log = logging.getLogger(__name__)
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super(MyWorkerClass, self).init_process() so that the ``run()``
loop is initiated.
"""
util.set_owner_process(self.cfg.uid, self.cfg.gid)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inherientence
[util.close_on_exec(s) for s in self.sockets]
util.close_on_exec(self.tmp.fileno())
self.log.close_on_exec()
self.init_signals()
self.wsgi = self.app.wsgi()
self.cfg.post_worker_init(self)
# Enter main run loop
self.booted = True
self.run()
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super(MyWorkerClass, self).init_process() so that the ``run()``
loop is initiated.
"""
util.set_owner_process(self.cfg.uid, self.cfg.gid)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inherientence
[util.close_on_exec(s) for s in self.sockets]
util.close_on_exec(self.tmp.fileno())
self.log.close_on_exec()
self.init_signals()
self.wsgi = self.app.wsgi()
# Enter main run loop
self.booted = True
self.run()
def __init__(self, workerid, ppid, socket, app, timeout, debug=False):
self.nr = 0
self.id = workerid
self.ppid = ppid
self.debug = debug
self.socket = socket
self.timeout = timeout
fd, tmpname = tempfile.mkstemp()
self.tmp = os.fdopen(fd, "r+b")
self.tmpname = tmpname
self.app = app
self.alive = True
self.log = logging.getLogger(__name__)
self.spinner = 0
# init pipe
self.PIPE = os.pipe()
map(util.set_non_blocking, self.PIPE)
map(util.close_on_exec, self.PIPE)
# prevent inherientence
util.close_on_exec(self.socket)
util.close_on_exec(fd)
self.address = self.socket.getsockname()
def close_on_exec(self):
for log in (self.error_log, self.access_log):
for handler in log.handlers:
if isinstance(handler, logging.FileHandler):
handler.acquire()
util.close_on_exec(handler.stream.fileno())
handler.release()
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super(MyWorkerClass, self).init_process() so that the ``run()``
loop is initiated.
"""
# set environment' variables
if self.cfg.env:
for k, v in self.cfg.env.items():
os.environ[k] = v
util.set_owner_process(self.cfg.uid, self.cfg.gid,
initgroups=self.cfg.initgroups)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inheritance
[util.close_on_exec(s) for s in self.sockets]
util.close_on_exec(self.tmp.fileno())
self.wait_fds = self.sockets + [self.PIPE[0]]
self.log.close_on_exec()
self.init_signals()
# start the reloader
if self.cfg.reload and self.cfg.reload != 'off':
def changed(fname):
self.log.info("Worker reloading: %s modified", fname)
self.alive = False
self.cfg.worker_int(self)
time.sleep(0.1)
sys.exit(0)
if self.cfg.reload == 'poll':
reloader_cls = Reloader
elif self.cfg.reload == 'inotify':
reloader_cls = InotifyReloader
else:
reloader_cls = preferred_reloader
self.reloader = reloader_cls(callback=changed)
self.reloader.start()
self.load_wsgi()
self.cfg.post_worker_init(self)
# Enter main run loop
self.booted = True
self.run()
def run(self):
# self.socket appears to lose its blocking status after
# we fork in the arbiter. Reset it here.
for s in self.sockets:
s.setblocking(0)
ready = self.sockets
while self.alive:
self.notify()
# Accept a connection. If we get an error telling us
# that no connection is waiting we fall down to the
# select which is where we'll wait for a bit for new
# workers to come give us some love.
for sock in ready:
try:
client, addr = sock.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(sock, client, addr)
# Keep processing clients until no one is waiting. This
# prevents the need to select() for every client that we
# process.
continue
except socket.error as e:
if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED,
errno.EWOULDBLOCK):
raise Exception("errno {0}: {1}".format(
e.args[0], e.args[1]))
# If our parent changed then we shut down.
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s", self)
return
try:
self.notify()
# if no timeout is given the worker will never wait and will
# use the CPU for nothing. This minimal timeout prevent it.
timeout = self.timeout or 0.5
ret = select.select(self.sockets, [], self.PIPE, timeout)
if ret[0]:
ready = ret[0]
continue
except select.error as e:
if e.args[0] == errno.EINTR:
ready = self.sockets
continue
if e.args[0] == errno.EBADF:
if self.nr < 0:
ready = self.sockets
continue
else:
return
raise
def accept(self, listener): # pragma: no cover
client, address = listener.accept()
# :class:`SyncIteratingWorker` uses non blocking connection sockets so we
# directly fall back on iteration when no data is available on connection
client.setblocking(False)
util.close_on_exec(client)
self.handle(listener, client, address)
def close_on_exec(self):
for log in (self.error_log, self.access_log):
for handler in log.handlers:
if isinstance(handler, logging.FileHandler):
handler.acquire()
util.close_on_exec(handler.stream.fileno())
handler.release()
def run(self):
# self.socket appears to lose its blocking status after
# we fork in the arbiter. Reset it here.
for s in self.sockets:
s.setblocking(0)
ready = self.sockets
while self.alive:
self.notify()
# Accept a connection. If we get an error telling us
# that no connection is waiting we fall down to the
# select which is where we'll wait for a bit for new
# workers to come give us some love.
for sock in ready:
try:
client, addr = sock.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(sock, client, addr)
# Keep processing clients until no one is waiting. This
# prevents the need to select() for every client that we
# process.
continue
except socket.error as e:
if e.args[0] not in (errno.EAGAIN, errno.ECONNABORTED,
errno.EWOULDBLOCK):
raise
# If our parent changed then we shut down.
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s", self)
return
try:
self.notify()
# if no timeout is given the worker will never wait and will
# use the CPU for nothing. This minimal timeout prevent it.
timeout = self.timeout or 0.5
ret = select.select(self.sockets, [], self.PIPE, timeout)
if ret[0]:
ready = ret[0]
continue
except select.error as e:
if e.args[0] == errno.EINTR:
ready = self.sockets
continue
if e.args[0] == errno.EBADF:
if self.nr < 0:
ready = self.sockets
continue
else:
return
raise
def handle(self, client, addr):
util.close_on_exec(client)
try:
req = http.HttpRequest(client, addr, self.address)
# WSGI 协议
response = self.app(req.read(), req.start_response)
http.HttpResponse(client, response, req).send()
except Exception, e:
self.log.exception("Error processing request. [%s]" % str(e))
def close_on_exec(self):
for log in loggers():
for handler in log.handlers:
if isinstance(handler, logging.FileHandler):
handler.acquire()
try:
if handler.stream:
util.close_on_exec(handler.stream.fileno())
finally:
handler.release()
def close_on_exec(self):
for log in loggers():
for handler in log.handlers:
if isinstance(handler, logging.FileHandler):
handler.acquire()
try:
if handler.stream:
util.close_on_exec(handler.stream.fileno())
finally:
handler.release()
def handle(self, client, addr):
util.close_on_exec(client)
try:
req = http.HttpRequest(client, addr, self.address)
response = self.app(req.read(), req.start_response)
http.HttpResponse(client, response, req).send()
except Exception, e:
# TODO: try to send something if an error happend
self.log.exception("Error processing request. [%s]" % str(e))
# try to send something if an error happend
util.close(client)
def run(self):
"Main master loop."
self.start()
util._setproctitle("master [%s]" % self.proc_name)
"Starting new customer process - dataset checker "
self.checker_alive = False
self.checker_pipe = os.pipe()
for p in self.checker_pipe:
util.set_non_blocking(p)
util.close_on_exec(p)
self.manage_checker()
try:
self.manage_workers()
while True:
self.maybe_promote_master()
sig = self.SIG_QUEUE.pop(0) if self.SIG_QUEUE else None
if sig is None:
self.sleep()
self.murder_workers()
self.manage_workers()
self.manage_checker(update_workers_pids=True)
continue
if sig not in self.SIG_NAMES:
self.log.info("Ignoring unknown signal: %s", sig)
continue
signame = self.SIG_NAMES.get(sig)
handler = getattr(self, "handle_%s" % signame, None)
if not handler:
self.log.error("Unhandled signal: %s", signame)
continue
self.log.info("Handling signal: %s", signame)
handler()
self.wakeup()
except StopIteration:
self.halt()
except KeyboardInterrupt:
self.halt()
except HaltServer as inst:
self.halt(reason=inst.reason, exit_status=inst.exit_status)
except SystemExit:
raise
except Exception:
self.log.info("Unhandled exception in main loop", exc_info=True)
self.stop(False)
if self.pidfile is not None:
self.pidfile.unlink()
sys.exit(-1)
def init_process(self):
util.set_owner_process(self.conf.uid, self.conf.gid)
# init pipe
self.PIPE = os.pipe()
map(util.set_non_blocking, self.PIPE)
map(util.close_on_exec, self.PIPE)
# prevent inherientence
util.close_on_exec(self.socket)
util.close_on_exec(self.fd)
self.init_signals()
def handle(self, client, addr):
util.close_on_exec(client)
try:
req = http.Request(client, addr, self.address, self.debug)
try:
response = self.app(req.read(), req.start_response)
except Exception, e:
util.write_error(client, traceback.format_exc())
return
http.Response(client, response, req).send()
def accept(self, listener):
"""listner -- a socket obj
"""
# Wait for an incoming connection. Return a new
# socket representing the connection, and the address
# of the client. For IP sockets, the address
# info is a pair (hostaddr, port).
client, addr = listener.accept()
# set or clear the blocking I/O flag
client.setblocking(1)
util.close_on_exec(client)
self.handle(listener, client, addr)
def accept(self, listener):
client, addr = listener.accept()
client.setblocking(1)
util.close_on_exec(client)
if self.stats:
self.stats.start_request()
self.notify()
self.handle(listener, client, addr)
if self.stats:
self.stats.end_request()
def run(self):
self.nr = 0
# self.socket appears to lose its blocking status after
# we fork in the arbiter. Reset it here.
self.socket.setblocking(0)
while self.alive:
self.nr = 0
self.notify()
# Accept a connection. If we get an error telling us
# that no connection is waiting we fall down to the
# select which is where we'll wait for a bit for new
# workers to come give us some love.
try:
client, addr = self.socket.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(client, addr)
self.nr += 1
except socket.error, e:
if e[0] not in (errno.EAGAIN, errno.ECONNABORTED):
raise
# Keep processing clients until no one is waiting. This
# prevents the need to select() for every client that we
# process.
if self.nr > 0:
continue
# If our parent changed then we shut down.
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s" % self)
return
try:
self.notify()
ret = select.select([self.socket], [], self.PIPE, self.timeout)
if ret[0]:
continue
except select.error, e:
if e[0] == errno.EINTR:
continue
if e[0] == errno.EBADF:
if self.nr < 0:
continue
else:
return
raise
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super(MyWorkerClass, self).init_process() so that the ``run()``
loop is initiated.
"""
# start the reloader
if self.cfg.reload:
def changed(fname):
self.log.info("Worker reloading: %s modified", fname)
os.kill(self.pid, signal.SIGTERM)
raise SystemExit()
Reloader(callback=changed).start()
# set environment' variables
if self.cfg.env:
for k, v in self.cfg.env.items():
os.environ[k] = v
util.set_owner_process(self.cfg.uid, self.cfg.gid)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inheritance
[util.close_on_exec(s) for s in self.sockets]
util.close_on_exec(self.tmp.fileno())
self.log.close_on_exec()
self.init_signals()
self.wsgi = self.app.wsgi()
self.cfg.post_worker_init(self)
# Enter main run loop
self.booted = True
self.run()
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super(MyWorkerClass, self).init_process() so that the ``run()``
loop is initiated.
"""
# start the reloader
if self.cfg.reload:
def changed(fname):
self.log.info("Worker reloading: %s modified", fname)
os.kill(self.pid, signal.SIGQUIT)
self.reloader = Reloader(callback=changed)
self.reloader.start()
# set environment' variables
if self.cfg.env:
for k, v in self.cfg.env.items():
os.environ[k] = v
util.set_owner_process(self.cfg.uid, self.cfg.gid)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inheritance
[util.close_on_exec(s) for s in self.sockets]
util.close_on_exec(self.tmp.fileno())
self.wait_fds = self.sockets + [self.PIPE[0]]
self.log.close_on_exec()
self.init_signals()
self.load_wsgi()
self.cfg.post_worker_init(self)
# Enter main run loop
self.booted = True
self.run()
def run(self):
# self.socket appears to lose its blocking status after
# we fork in the arbiter. Reset it here.
self.socket.setblocking(0)
while self.alive:
self.notify()
# Accept a connection. If we get an error telling us
# that no connection is waiting we fall down to the
# select which is where we'll wait for a bit for new
# workers to come give us some love.
try:
client, addr = self.socket.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(client, addr)
# Keep processing clients until no one is waiting. This
# prevents the need to select() for every client that we
# process.
continue
except socket.error, e:
if e[0] not in (errno.EAGAIN, errno.ECONNABORTED):
raise
# If our parent changed then we shut down.
if self.ppid != os.getppid():
self.log.info("Parent changed, shutting down: %s", self)
return
try:
self.notify()
ret = select.select([self.socket], [], self.PIPE, self.timeout)
if ret[0]:
continue
except select.error, e:
if e[0] == errno.EINTR:
continue
if e[0] == errno.EBADF:
if self.nr < 0:
continue
else:
return
raise
def _handle(self, sock):
try:
client, addr = sock.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(sock, client, addr)
# Keep processing clients until no one is waiting. This
# prevents the need to select() for every client that we
# process.
except socket.error as e:
if e.args[0] not in (
errno.EAGAIN,
errno.ECONNABORTED,
errno.EWOULDBLOCK
):
raise
def __init__(self, workerid, ppid, socket, app, timeout):
self.id = workerid
self.ppid = ppid
self.timeout = timeout
fd, tmpname = tempfile.mkstemp()
self.tmp = os.fdopen(fd, "r+b")
self.tmpname = tmpname
# prevent inherientence
self.socket = socket
util.close_on_exec(self.socket)
self.socket.setblocking(0)
util.close_on_exec(fd)
self.address = self.socket.getsockname()
self.app = app
self.alive = True
self.log = logging.getLogger(__name__)
def init_signals(self):
"""\
Initialize master signal handling. Most of the signals
are queued. Child signals only wake up the master.
"""
# close old PIPE
if self.PIPE:
[os.close(p) for p in self.PIPE]
# initialize the pipe
self.PIPE = pair = os.pipe()
for p in pair:
util.set_non_blocking(p)
util.close_on_exec(p)
self.log.close_on_exec()
# initialize all signals
[signal.signal(s, self.signal) for s in self.SIGNALS]
signal.signal(signal.SIGCHLD, self.handle_chld)
def init_signals(self):
"""\
Initialize master signal handling. Most of the signals
are queued. Child signals only wake up the master.
"""
# close old PIPE
if self.PIPE:
[os.close(p) for p in self.PIPE]
# initialize the pipe
self.PIPE = pair = os.pipe()
for p in pair:
util.set_non_blocking(p)
util.close_on_exec(p)
self.log.close_on_exec()
# initialize all signals
[signal.signal(s, self.signal) for s in self.SIGNALS]
signal.signal(signal.SIGCHLD, self.handle_chld)
def init_signals(self):
"""\
Initialize master signal handling. Most of the signals
are queued. Child signals only wake up the master.
"""
# close old PIPE
if self.PIPE:
[os.close(p) for p in self.PIPE]
# initialize the pipe
self.PIPE = pair = os.pipe()
for p in pair:
util.set_non_blocking(p)
util.close_on_exec(p)
self.log.close_on_exec()
# initialize all signals
[signal.signal(s, self.signal) for s in self.SIGNALS]
signal.signal(signal.SIGCHLD, self.handle_chld)
# Start periodical instrumentation
if self.use_statsd:
signal.alarm(STATSD_INTERVAL)
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super(MyWorkerClass, self).init_process() so that the ``run()``
loop is initiated.
"""
util.set_owner_process(self.cfg.uid, self.cfg.gid)
# For waking ourselves up
self.PIPE = os.pipe()
map(util.set_non_blocking, self.PIPE)
map(util.close_on_exec, self.PIPE)
# Prevent fd inherientence
util.close_on_exec(self.socket)
util.close_on_exec(self.fd)
self.init_signals()
self.wsgi = self.app.wsgi()
# Enter main run loop
self.booted = True
self.run()
def init_process(self):
if self.cfg.env:
for k, v in self.cfg.env.items():
os.environ[k] = v
util.set_owner_process(
self.cfg.uid,
self.cfg.gid,
initgroups=self.cfg.initgroups
)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inheritance
for s in self.sockets:
util.close_on_exec(s)
util.close_on_exec(self.tmp.fileno())
self.wait_fds = self.sockets + [self.PIPE[0]]
self.log.close_on_exec()
self.init_signals()
# start the reloader
if self.cfg.reload:
def changed(fname):
print(">>>> Worker restarted <<<<")
self.log.info("Worker reloading: %s modified", fname)
self.alive = False
self.cfg.worker_int(self)
time.sleep(0.1)
sys.exit(0)
reloader_cls = reloader_engines[self.cfg.reload_engine]
self.reloader = reloader_cls(
extra_files=self.cfg.reload_extra_files,
callback=changed
)
self.reloader.start()
self.load_wsgi()
self.cfg.post_worker_init(self)
# Enter main run loop
self.booted = True
self.run()
def init_process(self):
"""\
If you override this method in a subclass, the last statement
in the function should be to call this method with
super().init_process() so that the ``run()`` loop is initiated.
"""
self._log('init_process')
# set environment' variables
if self.cfg.env:
for k, v in self.cfg.env.items():
os.environ[k] = v
# 设置进程所属user group
util.set_owner_process(self.cfg.uid, self.cfg.gid, self.cfg.initgroups)
# Reseed the random number generator
util.seed()
# For waking ourselves up
self.PIPE = os.pipe()
for p in self.PIPE:
util.set_non_blocking(p)
util.close_on_exec(p)
# Prevent fd inheritance
for s in self.sockets:
util.close_on_exec(s)
util.close_on_exec(self.tmp.fileno())
self.wait_fds = self.sockets + [self.PIPE[0]]
self.log.close_on_exec()
self.init_signals()
self._start_reloader()
self.load_wsgi()
self.cfg.post_worker_init(self)
# Enter main run loop
self.booted = True
self.run()
def init_sock(self, sock):
sock.setblocking(1)
util.close_on_exec(sock)
def accept(self, listener):
client, addr = listener.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(listener, client, addr)
def accept(self, listener):
client, addr = listener.accept()
client.setblocking(1)
util.close_on_exec(client)
self.handle(listener, client, addr)