def test_parse_init_http():
u = "GET /test HTTP/1.1"
m, u, httpversion= http.parse_init_http(u)
assert m == "GET"
assert u == "/test"
assert httpversion == (1, 1)
assert not http.parse_init_http("invalid")
assert not http.parse_init_http("GET invalid HTTP/1.1")
assert not http.parse_init_http("GET /test foo/1.1")
def test_parse_init_http():
u = "GET /test HTTP/1.1"
m, u, httpversion= http.parse_init_http(u)
assert m == "GET"
assert u == "/test"
assert httpversion == (1, 1)
assert not http.parse_init_http("invalid")
assert not http.parse_init_http("GET invalid HTTP/1.1")
assert not http.parse_init_http("GET /test foo/1.1")
def _read_request_transparent(self, client_conn, scheme, host, port):
"""
Read a transparent HTTP request. Transparent means that the client isn't aware of proxying.
In other words, the client request starts with
"GET /foo.html HTTP/1.1"
rather than
"CONNECT example.com:80 HTTP/1.1"
"""
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn,httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def create_http_request(flowheader, reqbuf):
sfp = StringIO(reqbuf)
method, url, httpversion = http.parse_init_http(sfp.readline())
host, port, path = parse_url(url)
headers = http.read_headers(sfp)
if not host:
if not headers.get("host"):
host = flowheader.dstip
else:
host = headers.get("host")[0]
if ":" in host:
host = string.rsplit(host, ":", maxsplit=1)[0]
if port == None:
port = flowheader.dport
# TODO: passing None as the second arg will produce and error if "expect" is in the headers
content = http.read_http_body_request(sfp, None, headers, httpversion, None)
# content = http.read_http_body(sfp, headers, True, None)
return flow.Request(
None,
httpversion,
host,
port,
"http",
method,
path,
headers,
content,
flowheader.ts_request_start,
flowheader.ts_request_finish,
)
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
if headers is None:
raise ProxyError(400, "Invalid headers")
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
def read_request_transparent(self, client_conn):
orig = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not orig:
raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
self.log(client_conn, "transparent to %s:%s"%orig)
host, port = orig
if port in self.config.transparent_proxy["sslports"]:
scheme = "https"
if not self.ssl_established:
self.establish_ssl(client_conn, host, port)
else:
scheme = "http"
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn,httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def _read_request_origin_form(self, client_conn, scheme, host, port):
"""
Read a HTTP request with regular (origin-form) request line.
An example origin-form request line would be:
GET /foo.html HTTP/1.1
The request destination is already known from one of the following sources:
1) transparent proxy: destination provided by platform resolver
2) reverse proxy: fixed destination
3) regular proxy: known from CONNECT command.
"""
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
self.handle_expect_header(headers, httpversion)
content = http.read_http_body(
self.rfile, headers, self.config.body_size_limit, True
)
r = flow.Request(
client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
r.set_live(self.rfile, self.wfile)
return r
def read_request(self, client_conn):
self.rfile.reset_timestamps()
if self.config.transparent_proxy:
orig = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not orig:
raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
host, port = orig
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp())
def _read_request_origin_form(self, client_conn, scheme, host, port):
"""
Read a HTTP request with regular (origin-form) request line.
An example origin-form request line would be:
GET /foo.html HTTP/1.1
The request destination is already known from one of the following sources:
1) transparent proxy: destination provided by platform resolver
2) reverse proxy: fixed destination
3) regular proxy: known from CONNECT command.
"""
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, scheme,
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def handle_request(self):
"""
Returns True if handling should continue.
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
return
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s"%repr(line)
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
def read_request_proxy(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
if not self.proxy_connect_state:
connparts = http.parse_init_connect(line)
if connparts:
host, port, httpversion = connparts
headers = self.read_headers(authenticate=True)
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n'%self.server_version) +
'\r\n'
)
self.wfile.flush()
self.establish_ssl(client_conn, host, port)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
if self.proxy_connect_state:
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
host, port, _ = self.proxy_connect_state
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, "https", method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
else:
r = http.parse_init_proxy(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, scheme, host, port, path, httpversion = r
headers = self.read_headers(authenticate=True)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def read_request_reverse(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, "http",
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def read_request_reverse(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, "http", method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s"%repr(line)
self.info(s)
return False, dict(type = "error", msg = s)
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type = "error", msg = s)
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type = "error", msg = s)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
request_log = dict(
path=path,
method=method,
headers=headers.lst,
httpversion=httpversion,
sni=self.sni,
remote_address=self.client_address,
)
try:
content = http.read_http_body_request(self.rfile, self.wfile,
headers, httpversion, None)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type="error", msg=s)
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
elif self.config.reverse_proxy:
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
else:
line = self.get_line(self.rfile)
if line == "":
return None
if line.startswith("CONNECT"):
host, port, httpversion = http.parse_init_connect(line)
# FIXME: Discard additional headers sent to the proxy. Should I expose
# these to users?
while 1:
d = self.rfile.readline()
if d == '\r\n' or d == '\n':
break
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
certfile = self.find_cert(host, port, None)
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
if self.proxy_connect_state:
host, port, httpversion = self.proxy_connect_state
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
else:
method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
elif self.config.reverse_proxy:
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
else:
line = self.get_line(self.rfile)
if line == "":
return None
if line.startswith("CONNECT"):
r = http.parse_init_connect(line)
if not r:
)
self.wfile.flush()
if not self.server.ssloptions.not_after_connect:
try:
self.convert_to_ssl(
self.server.ssloptions.certfile,
self.server.ssloptions.keyfile,
)
except tcp.NetLibError, v:
s = str(v)
self.info(s)
return False, dict(type = "error", msg = s)
return True, None
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s"%repr(line)
self.info(s)
return False, dict(type = "error", msg = s)
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type = "error", msg = s)
clientcert = None
if self.clientcert:
clientcert = dict(
cert,
key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.
request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
)
except tcp.NetLibError, v:
s = str(v)
self.info(s)
return False, dict(type="error", msg=s)
return True, None
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
clientcert = None
if self.clientcert:
clientcert = dict(
dummycert = self.find_cert(client_conn, host, port, host)
sni = HandleSNI(self, client_conn, host, port, dummycert,
self.config.certfile or self.config.cacert)
try:
self.convert_to_ssl(dummycert,
self.config.certfile
or self.config.cacert,
handle_sni=sni)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, scheme,
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def read_request_proxy(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
m = utils.MemBool()
if m(http.parse_init_connect(line)):
headers = http.read_headers(self.rfile)
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
if not self.server.ssloptions.not_after_connect:
try:
cert, key, chain_file = self.server.ssloptions.get_cert(m.v[0])
self.convert_to_ssl(
cert, key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
)
except tcp.NetLibError as v:
s = str(v)
self.info(s)
return False, dict(type="error", msg=s)
return True, None
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
clientcert = None
if self.clientcert:
clientcert = dict(
cn=self.clientcert.cn,
subject=self.clientcert.subject,
serial=self.clientcert.serial,
notbefore=self.clientcert.notbefore.isoformat(),
notafter=self.clientcert.notafter.isoformat(),
keyinfo=self.clientcert.keyinfo,
)
retlog = dict(
type="crafted",
request=dict(
path=path,
method=method,
headers=headers.lst,
httpversion=httpversion,
sni=self.sni,
remote_address=self.address(),
clientcert=clientcert,
),
cipher=None,
)
if self.ssl_established:
retlog["cipher"] = self.get_current_cipher()
try:
content = http.read_http_body(
self.rfile, headers, None,
method, None, True
)
except http.HttpError as s:
s = str(s)
self.info(s)
return False, dict(type="error", msg=s)
for i in self.server.anchors:
if i[0].match(path):
self.info("crafting anchor: %s" % path)
again, retlog["response"] = self.serve_crafted(i[1])
return again, retlog
if not self.server.nocraft and path.startswith(self.server.craftanchor):
spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
self.info("crafting spec: %s" % spec)
try:
crafted = language.parse_response(spec)
except language.ParseException as v:
self.info("Parse error: %s" % v.msg)
crafted = language.make_error_response(
"Parse Error",
"Error parsing response spec: %s\n" % v.msg + v.marked()
)
again, retlog["response"] = self.serve_crafted(crafted)
return again, retlog
elif self.server.noweb:
crafted = language.make_error_response("Access Denied")
language.serve(crafted, self.wfile, self.server.request_settings)
return False, dict(
type="error",
msg="Access denied: web interface disabled"
)
else:
self.info("app: %s %s" % (method, path))
req = wsgi.Request("http", method, path, headers, content)
flow = wsgi.Flow(self.address, req)
sn = self.connection.getsockname()
a = wsgi.WSGIAdaptor(
self.server.app,
sn[0],
self.server.address.port,
version.NAMEVERSION
)
a.serve(flow, self.wfile)
return True, None