def _real_init(self, sock, basepath, module, args):
"""Actual initialization function. Broken out for error handling"""
setproctitle('zygote name=%s version=%s' % (self.name, self._version(),))
# Create a pipe(2) pair. This will be used so workers can detect when
# the intermediate zygote exits -- when this happens, a read event will
# happen on the read_pipe file descriptor, and the child can exit. We do
# this so that if the intermediate zygote exits unexpectedly for some
# reason, while it still has children workers running (which is an
# abnormal situation in and of itself), we aren't left with orphaned
# worker processes. Note that the write_pipe is normally never written
# on, we're just using this hack to get a read event on the read pipe.
self.read_pipe, self.write_pipe = os.pipe()
self.io_loop = ZygoteIOLoop(log_name='zygote.worker.ioloop')
os.chdir(basepath)
# Add basepath to sys.path so that application will be able to
# load what's required. We do this at zygote initialization
# to have the correct paths at new code reload.
site.addsitedir(os.path.realpath(basepath))
t = __import__(module, [], [], ['initialize', 'get_application'], 0)
self.sock = sock
self.get_application = t.get_application
set_nonblocking(self.control_socket)
self.io_loop.add_handler(self.control_socket.fileno(), self.handle_control, self.io_loop.READ)
self.notify_socket = AFUnixSender(self.io_loop)
self.notify_socket.connect('\0zygote_%d' % self.ppid)
signal.signal(signal.SIGCHLD, self.reap_child)
# If there is an initialize function defined then call it.
if hasattr(t, 'initialize'):
self.logger.info('initializing zygote')
t.initialize(*self.args)
if self.canary:
notify(self.notify_socket, message.MessageCanaryInit)
# Initialization is successful. This is not the canary zygote anymore.
self.canary = False
self.logger.info('new zygote started')
def _real_init(self, sock, basepath, module, args):
"""Actual initialization function. Broken out for error handling"""
self.version = basepath.split("/")[-1]
setproctitle("zygote version=%s" % (self.version,))
# Create a pipe(2) pair. This will be used so workers can detect when
# the intermediate zygote exits -- when this happens, a read event will
# happen on the read_pipe file descriptor, and the child can exit. We do
# this so that if the intermediate zygote exits unexpectedly for some
# reason, while it still has children workers running (which is an
# abnormal situation in and of itself), we aren't left with orphaned
# worker processes. Note that the write_pipe is normally never written
# on, we're just using this hack to get a read event on the read pipe.
self.read_pipe, self.write_pipe = os.pipe()
self.io_loop = ZygoteIOLoop(log_name="zygote.worker.ioloop")
os.chdir(basepath)
sys.path.insert(0, basepath)
t = __import__(module, [], [], ["initialize", "get_application"], 0)
self.sock = sock
self.get_application = t.get_application
set_nonblocking(self.control_socket)
self.io_loop.add_handler(self.control_socket.fileno(), self.handle_control, self.io_loop.READ)
self.notify_socket = AFUnixSender(self.io_loop)
self.notify_socket.connect("\0zygote_%d" % self.ppid)
signal.signal(signal.SIGCHLD, self.reap_child)
# If there is an initialize function defined then call it.
if hasattr(t, "initialize"):
self.logger.info("initializing zygote")
t.initialize(*self.args)
if self.canary:
notify(self.notify_socket, message.MessageCanaryInit)
# Initialization is successful. This is not the canary zygote anymore.
self.canary = False
self.logger.info("new zygote started")
def main(opts, extra_args):
setproctitle('zygote master %s' % (opts.name or opts.module, ))
zygote_logger = get_logger('zygote', opts.debug)
if not logging.root.handlers:
# XXX: WARNING
#
# We're disabling the root logger. Tornado's RequestHandler ONLY
# supports logging uncaught errors to the root logger. This will end
# poorly for you!
#
# We should probably provide a RequestHandler subclass that has
# _handle_request_exception overridden to do something useful.
# That might be hard to do without adding a tight version dependency
# on tornado.
logging.root.addHandler(NullHandler())
if opts.debug:
logging.root.setLevel(logging.DEBUG)
else:
logging.root.setLevel(logging.INFO)
zygote_logger.info('main started')
# Create the TCP listen socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
flags = fcntl.fcntl(sock.fileno(), fcntl.F_GETFD)
flags |= fcntl.FD_CLOEXEC
fcntl.fcntl(sock.fileno(), fcntl.F_SETFD, flags)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(0)
sock.bind((opts.interface, opts.port))
sock.listen(128)
ssl_options = None
if opts.cert:
ssl_options = dict(
certfile=opts.cert,
keyfile=opts.key,
ca_certs=opts.cacerts,
cert_reqs=ssl.CERT_OPTIONAL if opts.cacerts else ssl.CERT_NONE,
)
zygote_logger.info('using SSL with %s', ssl_options)
sock = ssl.wrap_socket(sock,
server_side=True,
do_handshake_on_connect=False,
**ssl_options)
master = ZygoteMaster(sock,
basepath=opts.basepath,
module=opts.module,
name=opts.name or opts.module,
version=opts.version,
num_workers=opts.num_workers,
control_port=opts.control_port,
control_socket_path=opts.control_socket_path,
application_args=extra_args,
max_requests=opts.max_requests,
zygote_base=opts.zygote_base,
ssl_options=ssl_options,
debug=opts.debug)
atexit.register(master.stop)
master.start()
def _initialize_worker(self, time_created):
# We're the child. We need to close the write_pipe in order for the
# read_pipe to get an event when the parent's write_pipe closes
# (otherwise the kernel is too smart and thinks that it's waiting
# for writes from *this* process' write_pipe).
os.close(self.write_pipe)
logger = get_logger('zygote.worker.worker_process')
logger.debug('new worker started')
def on_parent_exit(fd, events):
logger.error('detected that intermediate zygote died, exiting')
sys.exit(0)
# create a new i/o loop
del self.io_loop
io_loop = ZygoteIOLoop(log_name='zygote.worker.worker_process.ioloop')
# Install this worker's io_loop as the global io_loop; only applies in
# this fork. Programs that uses this io_loop instance should NOT use
# io_loop.start() because start() is invoked by the corresponding
# zygote worker.
if tornado.version_info >= (2,1,0):
io_loop.install()
else:
tornado.ioloop.IOLoop._instance = io_loop
# add the read pipe
io_loop.add_handler(self.read_pipe, on_parent_exit, io_loop.READ)
sock = AFUnixSender(io_loop, logger=logger)
sock.connect('\0zygote_%d' % self.ppid)
establish_signal_handlers(logger)
def on_headers(line, remote_ip, headers):
logger.debug('sending MessageHTTPBegin')
notify(sock, message.MessageHTTPBegin, "%s %s" % (remote_ip, line))
def on_close(disconnected=False):
logger.debug('sending MessageHTTPEnd')
notify(sock, message.MessageHTTPEnd)
notify(sock, message.MessageWorkerStart, '%d %d' % (int(time_created * 1e6), os.getppid()))
setproctitle('zygote-worker name=%s version=%s' % (self.name, self._version(),))
try:
# io_loop is passed into get_application for program to add handler
# or schedule task on the main io_loop. Program that uses this
# io_loop instance should NOT use io_loop.start() because start()
# is invoked by the corresponding zygote worker.
kwargs = {'io_loop': io_loop}
logger.debug("Invoking get_application")
app = self.get_application(*self.args, **kwargs)
except Exception:
logger.error("Unable to get application")
raise
# TODO: make keep-alive servers work
logger.debug("Creating HTTPServer")
http_server = HTTPServer(app,
io_loop=io_loop,
no_keep_alive=True,
close_callback=on_close,
headers_callback=on_headers,
ssl_options=self.ssl_options
)
if tornado.version_info >= (2,1,0):
http_server.add_socket(self.sock)
else:
http_server._socket = self.sock
io_loop.add_handler(self.sock.fileno(), http_server._handle_events, io_loop.READ)
logger.debug("Started ioloop...")
io_loop.start()
def main(opts, extra_args):
setproctitle('zygote master %s' % (opts.module,))
zygote_logger = get_logger('zygote', opts.debug)
if not logging.root.handlers:
# XXX: WARNING
#
# We're disabling the root logger. Tornado's RequestHandler ONLY
# supports logging uncaught errors to the root logger. This will end
# poorly for you!
#
# We should probably provide a RequestHandler subclass that has
# _handle_request_exception overridden to do something useful.
# That might be hard to do without adding a tight version dependency
# on tornado.
logging.root.addHandler(NullHandler())
if opts.debug:
logging.root.setLevel(logging.DEBUG)
else:
logging.root.setLevel(logging.INFO)
zygote_logger.info('main started')
# Create the TCP listen socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
flags = fcntl.fcntl(sock.fileno(), fcntl.F_GETFD)
flags |= fcntl.FD_CLOEXEC
fcntl.fcntl(sock.fileno(), fcntl.F_SETFD, flags)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.setblocking(0)
sock.bind((opts.interface, opts.port))
sock.listen(128)
ssl_options=None
if opts.cert:
ssl_options = dict(
certfile=opts.cert,
keyfile=opts.key,
ca_certs=opts.cacerts,
cert_reqs=ssl.CERT_OPTIONAL if opts.cacerts else ssl.CERT_NONE,
)
zygote_logger.info('using SSL with %s', ssl_options)
sock = ssl.wrap_socket(sock,
server_side=True,
do_handshake_on_connect=False,
**ssl_options
)
master = ZygoteMaster(
sock,
basepath=opts.basepath,
module=opts.module,
num_workers=opts.num_workers,
control_port=opts.control_port,
control_socket_path=opts.control_socket_path,
application_args=extra_args,
max_requests=opts.max_requests,
zygote_base=opts.zygote_base,
ssl_options=ssl_options,
debug=opts.debug
)
atexit.register(master.stop)
master.start()
