def __init__(self, conn, handle_request):
self.conn = conn
self.server_loop = conn.server_loop
self.max_header_line_size = self.server_loop.opts.max_header_line_size
self.scheme = 'http' if self.server_loop.ssl_context is None else 'https'
self.inheaders = MultiDict()
self.outheaders = MultiDict()
self.handle_request = handle_request
self.request_line = None
self.path = ()
self.qs = MultiDict()
"""When True, the request has been parsed and is ready to begin generating
the response. When False, signals the calling Connection that the response
should not be generated and the connection should close, immediately after
parsing the request."""
self.ready = False
"""Signals the calling Connection that the request should close. This does
not imply an error! The client and/or server may each request that the
connection be closed, after the response."""
self.close_connection = False
self.started_request = False
self.reponse_protocol = HTTP1
self.status_code = None
self.sent_headers = False
self.request_content_length = 0
self.chunked_read = False
def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri)
if path is None:
raise HTTPSimpleResponse(http_client.BAD_REQUEST, "No path component")
if b'#' in path:
raise HTTPSimpleResponse(http_client.BAD_REQUEST,
"Illegal #fragment in Request-URI.")
if scheme:
try:
scheme = scheme.decode('ascii')
except ValueError:
raise HTTPSimpleResponse(http_client.BAD_REQUEST,
'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2]
if parse_query:
try:
query = MultiDict.create_from_query_string(qs)
except Exception:
raise HTTPSimpleResponse(http_client.BAD_REQUEST,
'Unparseable query string')
else:
query = None
try:
path = '%2F'.join(
unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
raise HTTPSimpleResponse(http_client.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None,
(x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query
def parse_uri(uri, parse_query=True):
scheme, authority, path = parse_request_uri(uri)
if b'#' in path:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme:
try:
scheme = scheme.decode('ascii')
except ValueError:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Un-decodeable scheme')
path, qs = path.partition(b'?')[::2]
if parse_query:
try:
query = MultiDict.create_from_query_string(qs)
except Exception:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, 'Unparseable query string')
else:
query = None
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
raise HTTPSimpleResponse(httplib.BAD_REQUEST, as_unicode(e))
path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
return scheme, path, query
def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False)
request_body_file.seek(0)
outheaders = MultiDict()
data = RequestData(
self.method, self.path, self.query, inheaders, request_body_file,
outheaders, self.response_protocol, self.static_cache, self.opts,
self.remote_addr, self.remote_port, self.translator_cache, self.tdir
)
self.queue_job(self.run_request_handler, data)
def parse_request_line(self, buf, event, first=False): # {{{
line = self.readline(buf)
if line is None:
return
if line == b'\r\n':
# Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first:
return self.set_state(READ, self.parse_request_line, Accumulator())
return self.simple_response(httplib.BAD_REQUEST, 'Multiple leading empty lines not allowed')
try:
method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper()
except Exception:
return self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line")
if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST, "Unknown HTTP method")
try:
self.request_protocol = protocol_map[rp]
except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)]
scheme, authority, path = parse_request_uri(uri)
if b'#' in path:
return self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
if scheme:
try:
self.scheme = scheme.decode('ascii')
except ValueError:
return self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme')
qs = b''
if b'?' in path:
path, qs = path.split(b'?', 1)
try:
self.query = MultiDict.create_from_query_string(qs)
except Exception:
return self.simple_response(httplib.BAD_REQUEST, 'Unparseable query string')
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
return self.simple_response(httplib.BAD_REQUEST, as_unicode(e))
self.path = tuple(filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE
self.request_line = line.rstrip()
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(), Accumulator())
def __init__(self):
self.hdict = MultiDict()
self.lines = []
self.finished = False
class HTTPHeaderParser(object):
'''
Parse HTTP headers. Use this class by repeatedly calling the created object
with a single line at a time and checking the finished attribute. Can raise ValueError
for malformed headers, in which case you should probably return BAD_REQUEST.
Headers which are repeated are folded together using a comma if their
specification so dictates.
'''
__slots__ = ('hdict', 'lines', 'finished')
def __init__(self):
self.hdict = MultiDict()
self.lines = []
self.finished = False
def push(self, *lines):
for line in lines:
self(line)
def __call__(self, line):
'Process a single line'
def safe_decode(hname, value):
try:
return value.decode('utf-8')
except UnicodeDecodeError:
if hname in decoded_headers:
raise
return value
def commit():
if not self.lines:
return
line = b' '.join(self.lines)
del self.lines[:]
k, v = line.partition(b':')[::2]
key = normalize_header_name(k.strip().decode('ascii'))
val = safe_decode(key, v.strip())
if not key or not val:
raise ValueError('Malformed header line: %s' %
reprlib.repr(line))
if key in comma_separated_headers:
existing = self.hdict.pop(key)
if existing is not None:
val = existing + ', ' + val
self.hdict[key] = val
if self.finished:
raise ValueError('Header block already terminated')
if line == b'\r\n':
# Normal end of headers
commit()
self.finished = True
return
if line and line[0] in b' \t':
# It's a continuation line.
if not self.lines:
raise ValueError('Orphaned continuation line')
self.lines.append(line.lstrip())
else:
commit()
self.lines.append(line)
def __init__(self):
self.hdict = MultiDict()
self.lines = []
self.finished = False
class HTTPHeaderParser(object):
'''
Parse HTTP headers. Use this class by repeatedly calling the created object
with a single line at a time and checking the finished attribute. Can raise ValueError
for malformed headers, in which case you should probably return BAD_REQUEST.
Headers which are repeated are folded together using a comma if their
specification so dictates.
'''
__slots__ = ('hdict', 'lines', 'finished')
def __init__(self):
self.hdict = MultiDict()
self.lines = []
self.finished = False
def push(self, *lines):
for line in lines:
self(line)
def __call__(self, line):
'Process a single line'
def safe_decode(hname, value):
try:
return value.decode('utf-8')
except UnicodeDecodeError:
if hname in decoded_headers:
raise
return value
def commit():
if not self.lines:
return
line = b' '.join(self.lines)
del self.lines[:]
k, v = line.partition(b':')[::2]
key = normalize_header_name(k.strip().decode('ascii'))
val = safe_decode(key, v.strip())
if not key or not val:
raise ValueError('Malformed header line: %s' % reprlib.repr(line))
if key in comma_separated_headers:
existing = self.hdict.pop(key)
if existing is not None:
val = existing + ', ' + val
self.hdict[key] = val
if self.finished:
raise ValueError('Header block already terminated')
if line == b'\r\n':
# Normal end of headers
commit()
self.finished = True
return
if line and line[0] in b' \t':
# It's a continuation line.
if not self.lines:
raise ValueError('Orphaned continuation line')
self.lines.append(line.lstrip())
else:
commit()
self.lines.append(line)
def parse_request_line(self, buf, event, first=False): # {{{
line = self.readline(buf)
if line is None:
return
if line == b'\r\n':
# Ignore a single leading empty line, as per RFC 2616 sec 4.1
if first:
return self.set_state(READ, self.parse_request_line,
Accumulator())
return self.simple_response(
httplib.BAD_REQUEST,
'Multiple leading empty lines not allowed')
try:
method, uri, req_protocol = line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii').upper()
except Exception:
return self.simple_response(httplib.BAD_REQUEST,
"Malformed Request-Line")
if self.method not in HTTP_METHODS:
return self.simple_response(httplib.BAD_REQUEST,
"Unknown HTTP method")
try:
self.request_protocol = protocol_map[rp]
except KeyError:
return self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
self.response_protocol = protocol_map[min((1, 1), rp)]
scheme, authority, path = parse_request_uri(uri)
if b'#' in path:
return self.simple_response(httplib.BAD_REQUEST,
"Illegal #fragment in Request-URI.")
if scheme:
try:
self.scheme = scheme.decode('ascii')
except ValueError:
return self.simple_response(httplib.BAD_REQUEST,
'Un-decodeable scheme')
qs = b''
if b'?' in path:
path, qs = path.split(b'?', 1)
try:
self.query = MultiDict.create_from_query_string(qs)
except Exception:
return self.simple_response(httplib.BAD_REQUEST,
'Unparseable query string')
try:
path = '%2F'.join(
unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
return self.simple_response(httplib.BAD_REQUEST, as_unicode(e))
self.path = tuple(
filter(None, (x.replace('%2F', '/') for x in path.split('/'))))
self.header_line_too_long_error_code = httplib.REQUEST_ENTITY_TOO_LARGE
self.request_line = line.rstrip()
self.set_state(READ, self.parse_header_line, HTTPHeaderParser(),
Accumulator())
def prepare_response(self, inheaders, request_body_file):
if self.method == 'TRACE':
msg = force_unicode(self.request_line, 'utf-8') + '\n' + inheaders.pretty()
return self.simple_response(httplib.OK, msg, close_after_response=False)
request_body_file.seek(0)
outheaders = MultiDict()
data = RequestData(
self.method, self.path, self.query, inheaders, request_body_file,
outheaders, self.response_protocol, self.static_cache, self.opts,
self.remote_addr, self.remote_port
)
try:
output = self.request_handler(data)
except HTTP404 as e:
return self.simple_response(httplib.NOT_FOUND, msg=e.message or '', close_after_response=False)
output = self.finalize_output(output, data, self.method is HTTP1)
if output is None:
return
outheaders.set('Date', http_date(), replace_all=True)
outheaders.set('Server', 'calibre %s' % __version__, replace_all=True)
keep_alive = not self.close_after_response and self.opts.timeout > 0
if keep_alive:
outheaders.set('Keep-Alive', 'timeout=%d' % int(self.opts.timeout))
if 'Connection' not in outheaders:
if self.response_protocol is HTTP11:
if self.close_after_response:
outheaders.set('Connection', 'close')
else:
if not self.close_after_response:
outheaders.set('Connection', 'Keep-Alive')
ct = outheaders.get('Content-Type', '')
if ct.startswith('text/') and 'charset=' not in ct:
outheaders.set('Content-Type', ct + '; charset=UTF-8')
buf = [HTTP11 + (' %d ' % data.status_code) + httplib.responses[data.status_code]]
for header, value in sorted(outheaders.iteritems(), key=itemgetter(0)):
buf.append('%s: %s' % (header, value))
buf.append('')
self.response_ready(BytesIO(b''.join((x + '\r\n').encode('ascii') for x in buf)), output=output)
def read_request_line(self):
request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size)
# Set started_request to True so http_communicate() knows to send 408
# from here on out.
self.started_request = True
if not request_line:
return False
if request_line == b'\r\n':
# RFC 2616 sec 4.1: "...if the server is reading the protocol
# stream at the beginning of a message and receives a CRLF
# first, it should ignore the CRLF."
# But only ignore one leading line! else we enable a DoS.
request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size)
if not request_line:
return False
if not request_line.endswith(b'\r\n'):
self.simple_response(
httplib.BAD_REQUEST, "HTTP requires CRLF terminators")
return False
self.request_line = request_line
try:
method, uri, req_protocol = request_line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii')
except (ValueError, IndexError):
self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line")
return False
try:
self.request_protocol = protocol_map[rp]
except KeyError:
self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
return False
scheme, authority, path = parse_request_uri(uri)
if b'#' in path:
self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
return False
if scheme:
try:
self.scheme = scheme.decode('ascii')
except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme')
return False
qs = b''
if b'?' in path:
path, qs = path.split(b'?', 1)
try:
self.qs = MultiDict.create_from_query_string(qs)
except Exception:
self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line",
'Unparseable query string')
return False
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
self.simple_response(httplib.BAD_REQUEST, as_unicode(e))
return False
self.path = tuple(x.replace('%2F', '/') for x in path.split('/'))
self.response_protocol = protocol_map[min((1, 1), rp)]
return True
class HTTPPair(object):
''' Represents a HTTP request/response pair '''
def __init__(self, conn, handle_request):
self.conn = conn
self.server_loop = conn.server_loop
self.max_header_line_size = self.server_loop.opts.max_header_line_size
self.scheme = 'http' if self.server_loop.ssl_context is None else 'https'
self.inheaders = MultiDict()
self.outheaders = MultiDict()
self.handle_request = handle_request
self.request_line = None
self.path = ()
self.qs = MultiDict()
"""When True, the request has been parsed and is ready to begin generating
the response. When False, signals the calling Connection that the response
should not be generated and the connection should close, immediately after
parsing the request."""
self.ready = False
"""Signals the calling Connection that the request should close. This does
not imply an error! The client and/or server may each request that the
connection be closed, after the response."""
self.close_connection = False
self.started_request = False
self.reponse_protocol = HTTP1
self.status_code = None
self.sent_headers = False
self.request_content_length = 0
self.chunked_read = False
def parse_request(self):
"""Parse the next HTTP request start-line and message-headers."""
try:
if not self.read_request_line():
return
except MaxSizeExceeded:
self.simple_response(
httplib.REQUEST_URI_TOO_LONG,
"The Request-URI sent with the request exceeds the maximum allowed bytes.")
return
try:
if not self.read_request_headers():
return
except MaxSizeExceeded:
self.simple_response(
httplib.REQUEST_ENTITY_TOO_LARGE,
"The headers sent with the request exceed the maximum allowed bytes.")
return
self.ready = True
def read_request_line(self):
request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size)
# Set started_request to True so http_communicate() knows to send 408
# from here on out.
self.started_request = True
if not request_line:
return False
if request_line == b'\r\n':
# RFC 2616 sec 4.1: "...if the server is reading the protocol
# stream at the beginning of a message and receives a CRLF
# first, it should ignore the CRLF."
# But only ignore one leading line! else we enable a DoS.
request_line = self.conn.socket_file.readline(maxsize=self.max_header_line_size)
if not request_line:
return False
if not request_line.endswith(b'\r\n'):
self.simple_response(
httplib.BAD_REQUEST, "HTTP requires CRLF terminators")
return False
self.request_line = request_line
try:
method, uri, req_protocol = request_line.strip().split(b' ', 2)
rp = int(req_protocol[5]), int(req_protocol[7])
self.method = method.decode('ascii')
except (ValueError, IndexError):
self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line")
return False
try:
self.request_protocol = protocol_map[rp]
except KeyError:
self.simple_response(httplib.HTTP_VERSION_NOT_SUPPORTED)
return False
scheme, authority, path = parse_request_uri(uri)
if b'#' in path:
self.simple_response(httplib.BAD_REQUEST, "Illegal #fragment in Request-URI.")
return False
if scheme:
try:
self.scheme = scheme.decode('ascii')
except ValueError:
self.simple_response(httplib.BAD_REQUEST, 'Un-decodeable scheme')
return False
qs = b''
if b'?' in path:
path, qs = path.split(b'?', 1)
try:
self.qs = MultiDict.create_from_query_string(qs)
except Exception:
self.simple_response(httplib.BAD_REQUEST, "Malformed Request-Line",
'Unparseable query string')
return False
try:
path = '%2F'.join(unquote(x).decode('utf-8') for x in quoted_slash.split(path))
except ValueError as e:
self.simple_response(httplib.BAD_REQUEST, as_unicode(e))
return False
self.path = tuple(x.replace('%2F', '/') for x in path.split('/'))
self.response_protocol = protocol_map[min((1, 1), rp)]
return True
def read_request_headers(self):
# then all the http headers
try:
self.inheaders = read_headers(partial(self.conn.socket_file.readline, maxsize=self.max_header_line_size))
self.request_content_length = int(self.inheaders.get('Content-Length', 0))
except ValueError as e:
self.simple_response(httplib.BAD_REQUEST, as_unicode(e))
return False
if self.request_content_length > self.server_loop.opts.max_request_body_size:
self.simple_response(
httplib.REQUEST_ENTITY_TOO_LARGE,
"The entity sent with the request exceeds the maximum "
"allowed bytes (%d)." % self.server_loop.opts.max_request_body_size)
return False
# Persistent connection support
if self.response_protocol is HTTP11:
# Both server and client are HTTP/1.1
if self.inheaders.get("Connection", "") == "close":
self.close_connection = True
else:
# Either the server or client (or both) are HTTP/1.0
if self.inheaders.get("Connection", "") != "Keep-Alive":
self.close_connection = True
# Transfer-Encoding support
te = ()
if self.response_protocol is HTTP11:
rte = self.inheaders.get("Transfer-Encoding")
if rte:
te = [x.strip().lower() for x in rte.split(",") if x.strip()]
self.chunked_read = False
if te:
for enc in te:
if enc == "chunked":
self.chunked_read = True
else:
# Note that, even if we see "chunked", we must reject
# if there is an extension we don't recognize.
self.simple_response(httplib.NOT_IMPLEMENTED, "Unknown transfer encoding: %r" % enc)
self.close_connection = True
return False
if self.inheaders.get("Expect", '').lower() == "100-continue":
# Don't use simple_response here, because it emits headers
# we don't want.
msg = HTTP11 + " 100 Continue\r\n\r\n"
self.flushed_write(msg.encode('ascii'))
return True
def simple_response(self, status_code, msg=""):
abort = status_code in (httplib.REQUEST_ENTITY_TOO_LARGE, httplib.REQUEST_URI_TOO_LONG)
if abort:
self.close_connection = True
if self.reponse_protocol is HTTP1:
# HTTP/1.0 has no 413/414 codes
status_code = httplib.BAD_REQUEST
msg = msg.encode('utf-8')
buf = [
'%s %d %s' % (self.reponse_protocol, status_code, httplib.responses[status_code]),
"Content-Length: %s" % len(msg),
"Content-Type: text/plain; charset=UTF-8"
]
if abort and self.reponse_protocol is HTTP11:
buf.append("Connection: close")
buf.append('')
buf = [(x + '\r\n').encode('ascii') for x in buf]
buf.append(msg)
self.flushed_write(b''.join(buf))
def flushed_write(self, data):
self.conn.socket_file.write(data)
self.conn.socket_file.flush()
def repr_for_log(self):
return 'HTTPPair: %r\nPath:%r\nQuery:\n%s\nIn Headers:\n%s\nOut Headers:\n%s' % (
self.request_line, self.path, self.qs.pretty('\t'), self.inheaders.pretty('\t'), self.outheaders.pretty('\t')
)
def generate_static_output(self, name, generator):
return generate_static_output(self.server_loop.gso_cache, self.server_loop.gso_lock, name, generator)
def response(self):
if self.chunked_read:
self.input_reader = ChunkedReader(self.conn.socket_file, self.server_loop.opts.max_request_body_size)
else:
self.input_reader = FixedSizeReader(self.conn.socket_file, self.request_content_length)
output = self.handle_request(self)
if self.status_code is None:
raise Exception('Request handler did not set status_code')
# Read and discard any remaining body from the HTTP request
self.input_reader.read()
self.status_code, output = finalize_output(output, self.inheaders, self.outheaders, self.status_code)
self.send_headers()
if self.method != 'HEAD':
output.commit(self.conn.socket_file)
self.conn.socket_file.flush()
def send_headers(self):
self.sent_headers = True
