class WorkerProcess(object):
""" Worker process encapsulation.
These work a lot like server processes, except that they are
managed by daemon process as child processes.
The implementation uses two contexts:
- the server daemon context in which .start_server() and
.stop_server() are called
- the worker process contect in which .main() is run
The .main() method has to be overridden to implement the
worker process logic.
"""
# Note: This code is similar to ServerDaemon, but for worker
# processes we don't fork twice since we want the workers to be
# child processes of the server process.
# Name of the worker
name = 'Worker Process'
# PID of the worker process; set in both the server and the worker
# process context
pid = 0
# Started flag. Set by .start_worker()/.stop_worker() in the
# server context.
started = False
# Exit status code. Set by .worker_exited() in the server context.
exit_status = 0
# mxLog object to use. Inherited from the ServerDaemon if None
log = None
# Log id to use in the worker process. Inherited from the
# ServerDaemon if None
log_id = None
# Process name to use for the worker process. Note: this is not
# guaranteed to work. Inherited from the ServerDaemon if None
process_name = None
# Startup time of the worker processes in seconds. The
# .start_worker() method will wait this number of second for the
# worker process to start up.
worker_startup_time = 2
# Shutdown time of the worker processes in seconds. The
# .stop_worker() method will wait this number of second for the
# worker processes to terminate.
worker_shutdown_time = 2
# Kill time of the worker processes in seconds. The .stop_worker()
# method will wait this number of second for the worker processes
# to terminate after having received the KILL signal.
worker_kill_time = 1
# Range of file descriptors to close after the fork; all open fds
# except of stdin, stdout, stderr
close_file_descriptors = tuple(range(3, 99))
def __init__(self, server_daemon):
# Inherit settings from the server
if self.log is None:
self.log = server_daemon.log
if self.log is None:
self.log = Log.LogNothing
if self.log_id is None:
self.log_id = server_daemon.log_id
if self.process_name is None:
self.process_name = server_daemon.process_name
def __repr__(self):
return '%s(%s with PID %s)' % (self.__class__.__name__, self.name,
self.pid)
def setup_worker(self, **parameters):
""" Prepare the worker startup and adjust the parameters to be
passed on to the worker's .main() method.
This method is called by .start_worker() before forking
off a child process in order to give the WorkerProcess
implementation a chance to adjust itself to the
parameters.
It has to return a copy of the parameters keyword argument
dictionary.
This method is called in the context of the server.
"""
return parameters.copy()
def start_worker(self, **parameters):
""" Start the worker process and pass the given keyword parameters
to the .main() method.
"""
# Prepare startup
parameters = self.setup_worker(**parameters)
assert parameters is not None, \
'.setup_worker() did not return a parameters dictionary'
# Flush file descriptors
sys.stderr.flush()
sys.stdout.flush()
# Create a socket pair
server_socket, worker_socket = socket.socketpair(
socket.AF_UNIX, socket.SOCK_STREAM)
# Fork a child process, errors will be reported to the caller
pid = os.fork()
if pid != 0:
### Server process context ...
# Close our end of the socket pair
server_socket.close()
# Wait for the child to start up
worker_socket.settimeout(self.worker_startup_time)
try:
ok = worker_socket.recv(1)
except socket.timeout:
ok = None
worker_socket.close()
if not ok:
# Terminate the child, if it didn't startup in time
self.log(
self.log.ERROR, '%s: '
'Collecting worker process PID %s due to startup failure',
self.name, pid)
try:
self._kill_worker(pid)
except WorkerNotStoppedError:
pass
# Report the failure
raise WorkerNotStartedError(
'%s: Worker process with PID %s did not start up' %
(self.name, pid))
# Remember the worker process pid and return it
self.pid = pid
self.started = True
self.exit_status = 0
return pid
### Worker process context ...
# Close our end of the socket pair
worker_socket.close()
# Close all open fds except of stdin, stdout, stderr
self.log.close()
server_socket_fd = server_socket.fileno()
for i in self.close_file_descriptors:
if i == server_socket_fd:
# We'll close that manually later on
continue
try:
os.close(i)
except (IOError, OSError), reason:
pass
# Reopen the log file
self.log.open()
if self.log_id:
self.log.setup(log_id=self.log_id)
# Redirect stdout and stderr to the log file
self.log.redirect_stdout()
self.log.redirect_stderr()
# Try to rename the process
if self.process_name:
try:
Tools.setproctitle(self.process_name)
except AttributeError:
pass
# Set the PID of the worker process
self.pid = os.getpid()
# Let the server process know that we've started up
server_socket.send('1')
server_socket.close()
# Run the .main() method
rc = 0
try:
try:
self.log(self.log.INFO, '%s: Worker process PID %s %s',
self.name, self.pid, '-' * 40)
if _debug > 1:
self.log.object(
self.log.DEBUG,
'%s: Using the following startup parameters:' %
self.name, parameters)
# Run the worker's .main() method
main_rc = self.main(**parameters)
# Return the exit code, if it's an integer
if main_rc is not None and isinstance(main_rc, int):
rc = main_rc
except Exception:
# Something unexpected happened... log the problem and exit
self.log.traceback(self.log.ERROR, '%s: '
'Unexpected worker process error:',
self.name)
rc = 1
finally:
self.cleanup_worker()
# Exit process
os._exit(rc)
class ServerDaemon(object):
""" Server daemon encapsulation.
This class provides an easy way to setup a Unix server daemon
that uses a single process. It may still spawn off additional
processes, but this encapsulation only manages the main
process.
The implementation runs two contexts:
- the control context in which .start_server() and .stop_server()
are called
- the server process contect in which .main() is run
"""
# Name of the server
name = 'Server Daemon'
# PID of the process
pid = 0
# Location of the PID file of the parent process
pid_file = 'server.pid'
# umask to set for the forked server process
umask = 022
# Root dir to change to for the forked server process
root_dir = ''
# Range of file descriptors to close after the fork; all open fds
# except of stdin, stdout, stderr
close_file_descriptors = tuple(range(3, 99))
# mxLog object to use
log = Log.log
# Log id to use in the forked server process
log_id = ''
# Process name to use for the forked server process. Note: this is
# not guaranteed to work
process_name = ''
# Server startup time in seconds. The .start_server()
# method will wait at most this number of seconds for the main
# server process to initialize and enter the .main() method. This
# includes forking overhead, module import times, etc. It does not
# cover the startup time that the server may need to become usable
# for external applications. The startup time can be configured
# with .server_startup_time
server_startup_time = 2
# Startup initialization time of the server in seconds. The
# .start_server() method will unconditionally wait this number of
# seconds after having initialized the server in order to give the
# .main() method a chance to setup any resources it may need to
# initialize.
server_startup_init_time = 0
# Server shutdown time in seconds. The .stop_server()
# method will wait at most this number of seconds for the main
# server process to terminate after sending it a TERM signal.
server_shutdown_time = 2
# Kill time of the server processes in seconds. The .stop_server()
# method will wait this number of second for the worker processes
# to terminate after having received the KILL signal.
server_kill_time = 1
# Shutdown cleanup time of the server in seconds. The
# .stop_server() method will unconditionally wait this number of
# seconds after having terminated the main server process in order
# to give possibly additionally spawned processes a chance to
# terminate cleanly as well.
server_shutdown_cleanup_time = 0
###
def setup_server(self, **parameters):
""" Prepare the server startup and adjust the parameters to be
passed on to the server's .main() method.
This method is called by .start_server() before forking
off a child process in order to give the WorkerProcess
implementation a chance to adjust itself to the
parameters.
It has to return a copy of the parameters keyword argument
dictionary.
This method is called in the context of the server.
"""
return parameters.copy()
def _kill_server(self, pid):
""" Kill a server process pid and collect it.
Returns the process exit status or -1 in case this cannot
be determined.
Raises a ServerNotStoppedError in case the process cannot
be stopped.
"""
try:
return kill_process(pid,
shutdown_time=self.server_shutdown_time,
kill_time=self.server_kill_time,
log=self.log,
log_prefix='%s: ' % self.name)
except ProcessNotStoppedError:
# Did not work out...
raise ServerNotStoppedError(
'%s: Server process with PID %s did not stop' %
(self.name, pid))
def start_server(self, **parameters):
""" Starts the server.
Keyword parameters are passed on to the forked process'
.main() method.
Returns the PID of the started server daemon.
Raises a ServerAlreadyRunningError if the server is
already running. Raises a ServerNotStartedError in case
the daemon could not be started.
"""
# Verify if we have a running server process
pid = self.server_status()
if pid is not None:
raise ServerAlreadyRunningError(
'Server is already running (PID %s)' % pid)
# Prepare startup
parameters = self.setup_server(**parameters)
assert parameters is not None, \
'.setup_server() did not return a parameters dictionary'
# Flush the standard file descriptors
sys.stderr.flush()
sys.stdout.flush()
# Fork a child process, errors will be reported to the caller
pid = os.fork()
if pid != 0:
### Parent process
# Collect the first child
if _debug:
self.log(
self.log.DEBUG, '%s: '
'Waiting for the first child with PID %s to terminate',
self.name, pid)
os.waitpid(pid, 0)
# Wait a few seconds until the server has started
if _debug:
self.log(self.log.DEBUG, '%s: '
'Waiting for the server process to startup',
self.name)
for i in xrange(int(self.server_startup_time * 100) + 1):
spid = self.server_status()
if spid is not None:
break
time.sleep(0.01)
else:
# Server did not startup in time: terminate the first
# child
self.log(self.log.ERROR,
'%s: Server process failed to startup', self.name)
try:
self._kill_server(pid)
except ServerNotStoppedError:
pass
# Report the problem; XXX Note that the second child
# may still startup after this first has already
# terminated.
raise ServerNotStartedError('%s did not start up' % self.name)
if self.server_startup_init_time:
time.sleep(self.server_startup_init_time)
return spid
### This is the first child process
# Daemonize process
os.setpgrp()
if self.root_dir:
os.chdir(self.root_dir)
if self.umask:
os.umask(self.umask)
try:
# Try to become a session leader
os.setsid()
except OSError:
# We are already the process session leader
pass
# Close all open fds except of stdin, stdout, stderr
self.log.close()
for i in self.close_file_descriptors:
try:
os.close(i)
except (IOError, OSError), reason:
pass
# Fork again to become a separate daemon process
pid = os.fork()
if pid != 0:
# We need to terminate the "middle" process at this point, since we
# don't want to continue with two instances of the original caller.
# We must not call any cleanup handlers here.
os._exit(0)
### This is the second child process: the server daemon
# Turn the daemon into a process group leader
os.setpgrp()
# Reopen the log file
self.log.open()
if self.log_id:
self.log.setup(log_id=self.log_id)
# Redirect stdout and stderr to the log file
self.log.redirect_stdout()
self.log.redirect_stderr()
# Try to rename the process
if self.process_name:
try:
Tools.setproctitle(self.process_name)
except AttributeError:
pass
# Save the PID of the server daemon process
self.pid = os.getpid()
self.save_server_pid(self.pid)
# We need to remove the PID file on exit
rc = 0
try:
try:
self.log(self.log.INFO, '%s: Server process PID %s %s',
self.name, self.pid, '*' * 60)
# Run the server's .main() method
main_rc = self.main(**parameters)
# Return the exit code, if it's an integer
if main_rc is not None and isinstance(main_rc, int):
rc = main_rc
except SystemExit, exc:
# Normal shutdown
rc = exc.code
self.log(self.log.INFO, '%s: Shutting down with status: %s',
self.name, rc)
except Exception:
# Something unexpected happened... log the problem and exit
self.log.traceback(self.log.ERROR,
'%s: Unexpected server error:', self.name)
rc = 1
