def _read_headers(rfile):
"""
Read a set of headers.
Stop once a blank line is reached.
Returns:
A headers object
Raises:
exceptions.HttpSyntaxException
"""
ret = []
while True:
line = rfile.readline()
if not line or line == b"\r\n" or line == b"\n":
break
if line[0] in b" \t":
if not ret:
raise exceptions.HttpSyntaxException("Invalid headers")
# continued header
ret[-1] = (ret[-1][0], ret[-1][1] + b'\r\n ' + line.strip())
else:
try:
name, value = line.split(b":", 1)
value = value.strip()
if not name:
raise ValueError()
ret.append((name, value))
except ValueError:
raise exceptions.HttpSyntaxException(
"Invalid header line: %s" % repr(line))
return headers.Headers(ret)
def _read_chunked(rfile, limit=sys.maxsize):
"""
Read a HTTP body with chunked transfer encoding.
Args:
rfile: the input file
limit: A positive integer
"""
total = 0
while True:
line = rfile.readline(128)
if line == b"":
raise exceptions.HttpException("Connection closed prematurely")
if line != b"\r\n" and line != b"\n":
try:
length = int(line, 16)
except ValueError:
raise exceptions.HttpSyntaxException(
"Invalid chunked encoding length: {}".format(line))
total += length
if total > limit:
raise exceptions.HttpException(
"HTTP Body too large. Limit is {}, "
"chunked content longer than {}".format(limit, total))
chunk = rfile.read(length)
suffix = rfile.readline(5)
if suffix != b"\r\n":
raise exceptions.HttpSyntaxException("Malformed chunked body")
if length == 0:
return
yield chunk
def expected_http_body_size(
request: request.Request,
response: typing.Optional[response.Response] = None,
expect_continue_as_0: bool = True):
"""
Args:
- expect_continue_as_0: If true, incorrectly predict a body size of 0 for requests which are waiting
for a 100 Continue response.
Returns:
The expected body length:
- a positive integer, if the size is known in advance
- None, if the size in unknown in advance (chunked encoding)
- -1, if all data should be read until end of stream.
Raises:
exceptions.HttpSyntaxException, if the content length header is invalid
"""
# Determine response size according to
# http://tools.ietf.org/html/rfc7230#section-3.3
if not response:
headers = request.headers
if expect_continue_as_0 and headers.get("expect",
"").lower() == "100-continue":
return 0
else:
headers = response.headers
if request.method.upper() == "HEAD":
return 0
if 100 <= response.status_code <= 199:
return 0
if response.status_code == 200 and request.method.upper() == "CONNECT":
return 0
if response.status_code in (204, 304):
return 0
if "chunked" in headers.get("transfer-encoding", "").lower():
return None
if "content-length" in headers:
try:
sizes = headers.get_all("content-length")
different_content_length_headers = any(x != sizes[0]
for x in sizes)
if different_content_length_headers:
raise exceptions.HttpSyntaxException(
"Conflicting Content Length Headers")
size = int(sizes[0])
if size < 0:
raise ValueError()
return size
except ValueError as e:
raise exceptions.HttpSyntaxException(
"Unparseable Content Length") from e
if not response:
return 0
return -1
def _read_request_line(rfile):
try:
line = _get_first_line(rfile)
except exceptions.HttpReadDisconnect:
# We want to provide a better error message.
raise exceptions.HttpReadDisconnect("Client disconnected")
try:
method, path, http_version = line.split()
if path == b"*" or path.startswith(b"/"):
form = "relative"
scheme, host, port = None, None, None
elif method == b"CONNECT":
form = "authority"
host, port = _parse_authority_form(path)
scheme, path = None, None
else:
form = "absolute"
scheme, host, port, path = url.parse(path)
_check_http_version(http_version)
except ValueError:
raise exceptions.HttpSyntaxException(
"Bad HTTP request line: {}".format(line))
return form, method, scheme, host, port, path, http_version
def _read_request_line(rfile):
try:
line = _get_first_line(rfile)
except exceptions.HttpReadDisconnect:
# We want to provide a better error message.
raise exceptions.HttpReadDisconnect("Client disconnected")
try:
method, target, http_version = line.split()
if target == b"*" or target.startswith(b"/"):
scheme, authority, path = b"", b"", target
host, port = "", 0
elif method == b"CONNECT":
scheme, authority, path = b"", target, b""
host, port = url.parse_authority(authority, check=True)
if not port:
raise ValueError
else:
scheme, rest = target.split(b"://", maxsplit=1)
authority, path_ = rest.split(b"/", maxsplit=1)
path = b"/" + path_
host, port = url.parse_authority(authority, check=True)
port = port or url.default_port(scheme)
if not port:
raise ValueError
# TODO: we can probably get rid of this check?
url.parse(target)
_check_http_version(http_version)
except ValueError:
raise exceptions.HttpSyntaxException(f"Bad HTTP request line: {line}")
return host, port, method, scheme, authority, path, http_version
def expected_http_body_size(request, response=None):
"""
Returns:
The expected body length:
- a positive integer, if the size is known in advance
- None, if the size in unknown in advance (chunked encoding)
- -1, if all data should be read until end of stream.
Raises:
exceptions.HttpSyntaxException, if the content length header is invalid
"""
# Determine response size according to
# http://tools.ietf.org/html/rfc7230#section-3.3
if not response:
headers = request.headers
response_code = None
is_request = True
else:
headers = response.headers
response_code = response.status_code
is_request = False
if is_request:
if headers.get("expect", "").lower() == "100-continue":
return 0
else:
if request.method.upper() == "HEAD":
return 0
if 100 <= response_code <= 199:
return 0
if response_code == 200 and request.method.upper() == "CONNECT":
return 0
if response_code in (204, 304):
return 0
if "chunked" in headers.get("transfer-encoding", "").lower():
return None
if "content-length" in headers:
try:
size = int(headers["content-length"])
if size < 0:
raise ValueError()
return size
except ValueError:
raise exceptions.HttpSyntaxException("Unparseable Content Length")
if is_request:
return 0
return -1
def _parse_authority_form(hostport):
"""
Returns (host, port) if hostport is a valid authority-form host specification.
http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.1
Raises:
ValueError, if the input is malformed
"""
try:
host, port = hostport.split(b":")
port = int(port)
if not check.is_valid_host(host) or not check.is_valid_port(port):
raise ValueError()
except ValueError:
raise exceptions.HttpSyntaxException(
"Invalid host specification: {}".format(hostport))
return host, port
def _read_response_line(rfile):
try:
line = _get_first_line(rfile)
except exceptions.HttpReadDisconnect:
# We want to provide a better error message.
raise exceptions.HttpReadDisconnect("Server disconnected")
try:
parts = line.split(None, 2)
if len(parts) == 2: # handle missing message gracefully
parts.append(b"")
http_version, status_code, message = parts
status_code = int(status_code)
_check_http_version(http_version)
except ValueError:
raise exceptions.HttpSyntaxException(f"Bad HTTP response line: {line}")
return http_version, status_code, message
def _check_http_version(http_version):
if not re.match(br"^HTTP/\d\.\d$", http_version):
raise exceptions.HttpSyntaxException(
"Unknown HTTP version: {}".format(http_version))