class HTTPRequestRateController:
""" Implements rate limiting on a per ip address basis.
For this class to function properly handled http connections must be
registered through record_handled_connection() after workers have finished
with them.
Banned ip addresses can be checked through is_banned() method.
"""
max_recorded_addresses = 10
max_recorded_connections = 10
client_errors_threshold = config.getint('http', 'client_errors_threshold')
ban_duration = config.getint('http', 'ban_duration')
cleanup_threshold = 10000
required_records = client_errors_threshold * 2
def __init__(self):
self.bad_connection_records = {}
def is_banned(self, ip_address):
assert isinstance(ip_address, (str, bytes))
if ip_address not in self.bad_connection_records:
return False
record = self.bad_connection_records[ip_address]
if time.time() - record['first_recorded_on'] > self.ban_duration:
del self.bad_connection_records[ip_address]
return False
else:
return record['err_count'] >= self.client_errors_threshold
def record_handled_connection(self, ip_address, status_codes):
assert isinstance(ip_address, (str, bytes))
assert all(isinstance(sc, int) for sc in status_codes)
err_count = sum(sc in CONSIDERED_CLIENT_ERRORS for sc in status_codes)
if not err_count:
return
error_log.debug('Error count of address %s increased by %s.',
ip_address, err_count)
if len(self.bad_connection_records) == self.cleanup_threshold:
error_log.warning(
"Reached max recorded addresses for rate limiting "
"purposes. Dumping/freeing recorded bad connections.")
self.bad_connection_records = {}
if ip_address not in self.bad_connection_records:
self.bad_connection_records[ip_address] = dict(
ip_address=ip_address,
first_recorded_on=time.time(),
err_count=err_count)
else:
self.bad_connection_records[ip_address]['err_count'] += err_count
class WorkerProcess:
sigterm_timeout = config.getint('settings', 'process_sigterm_timeout')
def __init__(self, pid, fd_transport):
self.pid = pid
self.fd_transport = fd_transport
self.created_on = time.time()
self.sent_sockets = 0
self.sent_sigterm_on = None
self.terminating = False
def send_connections(self, connections):
sockets, addresses = zip(*connections)
msg = bytes(json.dumps(addresses), encoding='utf-8')
fds = [cs.fileno() for cs in sockets]
self.fd_transport.send_fds(msg=msg, fds=fds)
self.sent_sockets += len(sockets)
return True
def can_work(self):
return not self.terminating
def terminate(self):
if self.terminating:
return
try:
self.sent_sigterm_on = time.time()
os.kill(self.pid, ws.signals.SIGTERM)
self.terminating = True
except OSError as err:
error_log.warning(
'Failed to sent SIGTERM to worker with pid %s. '
'ERRNO=%s and MSG=%s', err.errno, err.strerror)
def kill_if_hanged(self, now=None):
if not self.terminating:
return False
now = now or time.time()
if now - self.sent_sigterm_on > self.sigterm_timeout:
# don't fail if worker is already dead.
try:
ws.signals.kill(self.pid, ws.signals.SIGKILL)
except OSError as err:
error_log.warning(
'Killing worker with pid %s failed. '
'ERRNO=%s and MSG=%s', err.errno, err.strerror)
return True
def close_ipc(self):
self.fd_transport.discard()
def __repr__(self):
return 'WorkerProcess(pid={})'.format(self.pid)
def recv_request(sock,
chunk_size=4096,
timeout=config.getint('http', 'connection_timeout'),
connection_timeout=config.getint('http', 'request_timeout')):
error_log.debug3('recv_request() from %s', sock)
assert isinstance(sock, ws.sockets.Socket)
assert isinstance(chunk_size, int)
total_length = 0
chunks = []
body_offset = -1
start = time.time()
sock.settimeout(timeout)
while body_offset == -1:
if total_length > MAX_HTTP_META_LEN:
raise ParserException(code='PARSER_REQUEST_TOO_LONG')
elif time.time() - start > connection_timeout:
raise ParserException(code='CS_PEER_CONNECTION_TIMED_OUT')
try:
chunk = sock.recv(chunk_size)
except ws.sockets.TimeoutException:
raise ParserException(code='CS_PEER_SEND_IS_TOO_SLOW')
if not chunk:
raise ParserException(code='CS_PEER_NOT_SENDING')
chunks.append(chunk)
total_length += len(chunk)
body_offset = chunk.find(b'\r\n\r\n')
lines = []
leftover_body = b''
for i, chunk in enumerate(chunks):
if i == len(chunks) - 1:
line_chunk = chunk[:body_offset]
leftover_body = chunk[body_offset:]
else:
line_chunk = chunk
lines.extend(line_chunk.split(b'\r\n'))
return lines, leftover_body
def __init__(self):
if not sys_has_fork_support():
raise SysError(code='FORK_NOT_IMPLEMENTED',
msg="Kernel or C lib versions don't have "
"fork() support.")
self.host = config['settings']['host']
self.port = config.getint('settings', 'port')
self.tcp_backlog_size = config.getint('settings', 'tcp_backlog_size')
self.process_count_limit = config.getint('settings',
'process_count_limit')
self.execution_context = self.ExecutionContext.main
self.sock = ws.sockets.ServerSocket(ws.sockets.AF_INET,
ws.sockets.SOCK_STREAM)
self.sock.setsockopt(ws.sockets.SOL_SOCKET, ws.sockets.SO_REUSEADDR, 1)
self.accepted_connections = 0
self.workers = {}
self.reaping = False
self.reaped_pids = set()
self.received_signals = set()
ws.signals.signal(ws.signals.SIGCHLD, self.reap_children)
ws.signals.signal(ws.signals.SIGUSR1, self.receive_signal)
def build_response(status_code,
*,
body=b'',
reason_phrase='',
headers=None,
version='HTTP/1.1'):
assert isinstance(status_code, int)
status_line = HTTPStatusLine(http_version=version,
status_code=status_code,
reason_phrase=reason_phrase)
headers = HTTPHeaders(headers or {})
if body:
assert 'Content-Length' in headers
else:
headers['Content-Length'] = 0
if status_code == 503:
headers['Retry-After'] = config.getint('http', 'retry_after') * 2
return HTTPResponse(status_line=status_line, headers=headers, body=body)
class SocketIterator:
""" Optimal byte iterator over plain sockets.
The __next__ method of this class ALWAYS returns one byte from the
underlying socket or raises an exception.
Exceptions raised during iteration:
ClientSocketException(code='CS_PEER_SEND_IS_TOO_SLOW') - when __next__ is
called and the socket times out.
ClientSocketException(code='CS_PEER_NOT_SENDING' - when __next__ is called
and the client sends 0 bytes through the socket indicating he is done.
ClientSocketException(code='CS_CONNECTION_TIMED_OUT') - when __next__ is
called but the connection_timeout has been exceeded.
StopIteration() - if __next__ is called after the socket was broken
"""
buffer_size = 2048
default_socket_timeout = config.getint('http', 'request_timeout')
default_connection_timeout = config.getint('http', 'connection_timeout')
def __init__(self,
sock,
*,
socket_timeout=default_socket_timeout,
connection_timeout=default_connection_timeout):
assert isinstance(sock, (ws.sockets.Socket, ws.sockets.SSLSocket))
assert isinstance(socket_timeout, int)
assert isinstance(connection_timeout, int)
self.sock = sock
self.current_chunk = None
self.socket_broke = False
self.connection_timeout = connection_timeout
self.connected_on = time.time()
self.written = False
self.sock.settimeout(socket_timeout)
def __iter__(self):
return self
def __next__(self):
if self.current_chunk:
try:
return next(self.current_chunk)
except StopIteration:
pass
elif self.socket_broke:
raise StopIteration()
if self.connected_on + self.connection_timeout < time.time():
raise ClientSocketException(code='CS_CONNECTION_TIMED_OUT')
try:
chunk = self.sock.recv(self.__class__.buffer_size)
except ws.sockets.TimeoutException as e:
error_log.warning('Socket timed out while receiving request.')
raise ClientSocketException(code='CS_PEER_SEND_IS_TOO_SLOW') from e
error_log.debug3('Read chunk %s', chunk)
if chunk == b'':
error_log.info('Socket %d broke', self.sock.fileno())
self.socket_broke = True
raise ClientSocketException(code='CS_PEER_NOT_SENDING')
self.current_chunk = iter(chunk)
return next(self.current_chunk)
import re
import time
import ws.http.structs
import ws.utils
import ws.sockets
from ws.config import config
from ws.err import *
from ws.logs import error_log
MAX_HTTP_META_LEN = config.getint('http', 'max_http_meta_len')
ALLOWED_HTTP_METHODS = frozenset(
{'HEAD', 'GET', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE'})
class ParserException(ServerException):
default_msg = 'Failed to parse request due to bad syntax.'
default_code = 'PARSER_BAD_SYNTAX'
def __init__(self, msg=default_msg, code=default_code):
super().__init__(msg=msg, code=code)
class ClientSocketException(ParserException):
default_msg = 'Client socket caused an error.'
default_code = 'CS_ERROR'
def __init__(self, msg=default_code, code=default_code):
super().__init__(msg=msg, code=code)
import time
import ws.auth
import ws.cgi
import ws.http.parser
import ws.http.structs
import ws.http.utils
import ws.ratelimit
import ws.serve
import ws.sockets
from ws.config import config
from ws.err import *
from ws.http.utils import request_is_persistent, response_is_persistent
from ws.logs import error_log
CLIENT_ERRORS_THRESHOLD = config.getint('http', 'client_errors_threshold')
exc_handler = ExcHandler()
# noinspection PyUnusedLocal
@exc_handler(AssertionError)
def server_err_handler(exc):
error_log.exception('Internal server error.')
return ws.http.utils.build_response(500), True
# noinspection PyUnusedLocal
@exc_handler(PeerError)
def peer_err_handler(exc):
error_log.warning('PeerError occurred. msg={exc.msg} code={exc.code}'