def work(self):
error_log.info('Entering endless loop of processing sockets.')
while True:
sock, address = (None, None)
# noinspection PyBroadException
try:
if not self.connections:
self.recv_new_sockets()
sock, address = self.connections.popleft()
self.handle_connection(socket=sock, address=address)
except SignalReceivedException as err:
if err.signum == ws.signals.SIGTERM:
error_log.info('Breaking work() loop due to signal %s.',
ws.signals.Signals(err.signum).name)
break
else:
error_log.exception('Unknown signal during work() loop')
except KeyboardInterrupt:
break
except Exception:
error_log.exception('Exception occurred during work() loop.')
continue
finally:
if sock:
sock.shutdown(ws.sockets.SHUT_RDWR, pass_silently=True)
sock.close(pass_silently=True)
# noinspection PyUnreachableCode
self.cleanup()
return 0
def cleanup(self):
""" Closing listening socket and reap children.
This method sleeps for the maximum timeout of SIGTERM signal sent to
a worker. (Worker.sigterm_timeout)
"""
# don't cleanup workers because their sockets were already closed
# during the self.fork_worker() call
if self.execution_context == self.ExecutionContext.worker:
return
error_log.info("Closing server's listening socket")
try:
self.sock.close()
except OSError:
error_log.exception('close() on listening socket failed.')
ws.signals.signal(ws.signals.SIGCHLD, ws.signals.SIG_DFL)
active_workers = [
worker for worker in self.workers.values()
if worker.pid not in self.reaped_pids
]
for worker in active_workers:
worker.terminate()
if active_workers:
timeout = max(worker.sigterm_timeout for worker in active_workers)
error_log.info('Waiting %s seconds for children to finish.',
timeout)
time.sleep(timeout)
for worker in active_workers:
pid, exit = os.waitpid(worker.pid, os.WNOHANG)
if not pid:
worker.kill_if_hanged()
def __next__(self):
if self.current_chunk:
try:
return next(self.current_chunk)
except StopIteration:
pass
elif self.socket_broke:
raise StopIteration()
if self.connected_on + self.connection_timeout < time.time():
raise ClientSocketException(code='CS_CONNECTION_TIMED_OUT')
try:
chunk = self.sock.recv(self.__class__.buffer_size)
except ws.sockets.TimeoutException as e:
error_log.warning('Socket timed out while receiving request.')
raise ClientSocketException(code='CS_PEER_SEND_IS_TOO_SLOW') from e
error_log.debug3('Read chunk %s', chunk)
if chunk == b'':
error_log.info('Socket %d broke', self.sock.fileno())
self.socket_broke = True
raise ClientSocketException(code='CS_PEER_NOT_SENDING')
self.current_chunk = iter(chunk)
return next(self.current_chunk)
def listen(self):
assert all(isinstance(w, WorkerProcess) for w in self.workers.values())
assert len(self.workers) == self.process_count_limit
error_log.info('Listening with backlog %s...', self.tcp_backlog_size)
self.sock.listen(self.tcp_backlog_size)
while True:
# TODO add rate limiting on rate of clients sending connections
# instead of on the HTTP syntax (which will be deferred to workers)
try:
sock, address = self.sock.accept()
except OSError as err:
error_log.warning('accept() raised ERRNO=%s with MSG=%s',
err.errno, err.strerror)
# TODO perhaps reopen failed listening sockets.
assert err.errno not in (errno.EBADF, errno.EFAULT,
errno.EINVAL, errno.ENOTSOCK,
errno.EOPNOTSUPP)
# don't break the listening loop just because one accept failed
continue
self.accepted_connections += 1
passed = self.distribute_connection(client_socket=sock,
address=address)
if not passed:
# TODO fork and reply quickly with a 503
error_log.warning(
'Could not distribute connection %s / %s to '
'workers. Dropping connection.', sock, address)
sock.close(pass_silently=True)
# duplicate the set so SIGCHLD handler doesn't cause problems
to_remove = frozenset(self.reaped_pids)
for pid in to_remove:
old_worker = self.workers.pop(pid)
old_worker.close_ipc()
self.reaped_pids.remove(pid)
# call outside of loop to avoid a race condition where a
# worker_process get's forked with a pid in self.reaped_pids
missing = self.process_count_limit - len(self.workers)
if missing > 0:
self.fork_workers(missing)
for worker_process in self.workers.values():
worker_process.kill_if_hanged()
if ws.signals.SIGUSR1 in self.received_signals:
for pid in self.workers:
ws.signals.kill(pid, ws.signals.SIGUSR1)
self.received_signals.remove(ws.signals.SIGUSR1)
def reindex_files(self):
error_log.info('Reindexing files under dir %s', self.document_root)
file_keys = set()
for dir_path, dir_names, file_names in os.walk(self.document_root):
for fn in file_names:
fp = os.path.join(dir_path, fn)
stat = os.stat(fp)
file_keys.add((stat.st_ino, stat.st_dev))
self.file_keys = frozenset(file_keys)
error_log.debug3('Indexed file keys are: %s', self.file_keys)
self.reindex_is_scheduled = False
def handle_connection(self,
socket,
address,
quick_reply_with=None,
ssl_only=config.getboolean('ssl', 'strict')):
assert isinstance(socket, ws.sockets.Socket)
assert isinstance(address, collections.Sequence)
error_log.debug3('handle_connection()')
if self.rate_controller.is_banned(address[0]):
socket.close(pass_silently=True)
return
wrapped_sock = socket
if self.ssl_ctx:
if socket.client_uses_ssl():
wrapped_sock = ws.sockets.SSLSocket.from_sock(
sock=socket, context=self.ssl_ctx, server_side=True)
elif ssl_only:
quick_reply_with = hutils.build_response(403)
else:
error_log.info('Client on %s / %s does not use SSL/TLS',
socket, address)
conn_worker = ws.cworker.ConnectionWorker(
sock=wrapped_sock,
address=address,
auth_scheme=self.auth_scheme,
static_files=self.static_files,
worker_ctx={'request_stats': self.request_stats})
try:
with conn_worker:
conn_worker.process_connection(
quick_reply_with=quick_reply_with)
finally:
self.rate_controller.record_handled_connection(
ip_address=address[0], status_codes=conn_worker.status_codes())
for exchange_stats in conn_worker.generate_stats():
for stat_name, val in exchange_stats.items():
self.request_stats[stat_name]['total'] += val
self.request_stats[stat_name]['count'] += 1
for exchange in conn_worker.exchanges:
access_log.log(**exchange)
def cleanup(self):
error_log.info('Cleaning up... %s total leftover connections.',
len(self.connections))
self.fd_transport.discard()
for sock, address in self.connections:
# noinspection PyBroadException
try:
res = hutils.build_response(503)
self.handle_connection(socket=sock,
address=address,
quick_reply_with=res)
except Exception:
error_log.exception(
'Error while cleaning up client on '
'%s / %s', sock, address)
finally:
sock.close(pass_silently=True)
def profile(enabled=True):
if not enabled:
yield
return
error_log.info('Starting profiling.')
profiler = cProfile.Profile()
profiler.enable()
profile_log.profile('Enabled profiling')
try:
yield
finally:
profiler.disable()
profile_log.profile('Disabled profiling')
s = io.StringIO()
ps = pstats.Stats(profiler, stream=s)
ps = ps.sort_stats('cumulative')
ps.print_stats()
profile_log.profile('cProfiler results:\n %s', s.getvalue())
def sys_has_fork_support():
error_log.info('Checking if system has fork support by doing a '
'dummy fork...')
try:
pid = os.fork()
except OSError as err:
if err.errno == errno.ENOSYS:
error_log.critical('System does not have fork() support.')
return False
else:
return True
if pid == 0:
# noinspection PyProtectedMember
os._exit(0)
else:
error_log.info('Fork successful. Cleaning up dummy child '
'(pid={:d})...'.format(pid))
os.waitpid(pid, 0)
return True
def fork_workers(self, count=1):
error_log.info('Forking %s workers', self.process_count_limit)
assert isinstance(count, int)
for _ in range(count):
fd_transport = ws.sockets.FDTransport()
pid = os.fork()
if pid:
self.execution_context = self.ExecutionContext.main
error_log.debug('Forked worker with pid=%s', pid)
ws.sockets.randomize_ssl_after_fork()
fd_transport.mode = 'sender'
wp = WorkerProcess(pid=pid, fd_transport=fd_transport)
self.workers[wp.pid] = wp
else:
self.execution_context = self.ExecutionContext.worker
ws.signals.reset_handlers(excluding={ws.signals.SIGTERM})
ws.signals.signal(ws.signals.SIGCHLD, ws.signals.SIG_IGN)
# noinspection PyBroadException
try:
ws.logs.setup_worker_handlers()
fd_transport.mode = 'receiver'
for other_worker in self.workers.values():
other_worker.close_ipc()
self.sock.close()
os.close(0)
os.close(1)
os.close(2)
with profile(WORKER_PROFILING_ON):
worker = ws.worker.Worker(fd_transport=fd_transport)
exit_code = worker.work()
except BaseException:
error_log.exception('Worker failed.')
exit_code = 1
# noinspection PyProtectedMember
os._exit(exit_code)
def main():
# Main process should never exit from the server.listen() loop unless
# an exception occurs.
fd_limit = subprocess.check_output(['ulimit', '-n'], shell=True)
error_log.info('ulimit -n is "%s"', fd_limit.decode('ascii'))
server = Server()
server.setup()
with profile(SERVER_PROFILING_ON):
# noinspection PyBroadException
try:
server.listen()
except SignalReceivedException as err:
if err.signum == ws.signals.SIGTERM:
error_log.info('SIGTERM signal broke listen() loop.')
else:
error_log.exception('Unknown signal broke listen() loop.')
except KeyboardInterrupt:
error_log.info('KeyboardInterrupt broke listen() loop.')
except BaseException:
error_log.exception('Unhandled exception broke listen() loop.')
finally:
server.cleanup()
def raising_signal_handler(signum, stack_info):
error_log.info('Received signum %s in raising signal handler.', signum)
signame = siglib.Signals(signum).name
raise SignalReceivedException(msg='Received signal {}'.format(signame),
code='DEFAULT_HANDLER_CAUGHT_SIGNAL',
signum=signum)
def kill(pid, signum):
signame = siglib.Signals(signum).name
error_log.info('Sending %s to pid %s.', signame, pid)
os.kill(pid, signum)
def signal(signum, handler):
signame = siglib.Signals(signum).name
error_log.info('Setting handler of %s to %s', signame, handler)
siglib.signal(signum, handler)
def execute_script(request, socket, body_start=b''):
assert isinstance(socket, (ws.sockets.SSLSocket, ws.sockets.Socket))
assert isinstance(body_start, (bytes, bytearray))
assert can_handle_request(request)
uri = request.request_line.request_target
cgi_script = find_cgi_script(uri)
error_log.info('Executing CGI script %s', cgi_script.name)
script_env = prepare_cgi_script_env(request, socket)
error_log.debug('CGIScript environment will be: %s', script_env)
has_body = 'Content-Length' in request.headers
try:
proc = subprocess.Popen(
args=os.path.abspath(cgi_script.script_path),
env=script_env,
stdin=subprocess.PIPE if has_body else socket.fileno(),
stdout=socket.fileno())
except (OSError, ValueError):
error_log.exception(
'Failed to open subprocess for cgi script {}'.format(
cgi_script.name))
return ws.http.utils.build_response(500)
# noinspection PyBroadException
try:
if not has_body:
return
error_log.debug('Request to CGI has body. Writing to stdin...')
start = time.time()
length = int(request.headers['Content-Length'])
read_bytes = 0
chunk_size = 4096
timeout_reached = False
while read_bytes < length and not timeout_reached:
if body_start:
chunk = body_start
body_start = b''
else:
chunk = socket.recv(chunk_size)
read_bytes += len(chunk)
if not chunk:
break
while chunk:
avail = select.select([], [proc.stdin], [], cgi_script.timeout)
_, wlist, _ = avail
if wlist:
written = wlist[0].write(chunk)
chunk = chunk[written:]
else:
timeout_reached = True
break
if time.time() - start > cgi_script.timeout:
timeout_reached = True
if timeout_reached:
error_log.warning(
'CGI script %s took too long to read body. '
'Leaving process alive but no more data will '
'be piped to it.', cgi_script)
except ws.sockets.TimeoutException:
error_log.warning("Client sent data too slowly - socket timed out.")
except Exception:
error_log.exception('Failed to write body to CGI script.')
finally:
try:
proc.stdin.close()
except OSError as err:
error_log.warning(
'Closing CGI stdin pipe failed. ERRNO=%s MSG=%s.', err.errno,
err.strerror)
def setup(self):
""" Bind socket and pre-fork workers. """
error_log.info('Binding server on %s:%s', self.host, self.port)
self.sock.bind((self.host, self.port))
self.fork_workers(self.process_count_limit)
import io
import os
import re
import ws.http.utils
from ws.config import config
from ws.err import *
from ws.http.structs import HTTPResponse, HTTPStatusLine, HTTPHeaders
from ws.logs import error_log
STATIC_ROUTE = config['routes']['static']
STATIC_DIR = os.path.realpath(os.path.abspath(
config['resources']['static_dir']
))
error_log.info('Configured static route is %s. Directory is %s',
STATIC_ROUTE, STATIC_DIR)
if not STATIC_ROUTE.endswith('/'):
raise SysError(msg="routes.static must end with a '/'",
code='CONFIG_BAD_STATIC_ROUTE')
if not os.path.isdir(STATIC_DIR):
raise SysError(msg='resources.static_dir field must be a directory',
code='CONFIG_BAD_STATIC_DIR')
class StaticFiles:
def __init__(self, document_root=STATIC_DIR, route_prefix=STATIC_ROUTE):
self.document_root = document_root
self.route_prefix = route_prefix
self.file_keys = frozenset()
self.reindex_is_scheduled = False
