def read_request_proxy(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
if not self.proxy_connect_state:
connparts = http.parse_init_connect(line)
if connparts:
host, port, httpversion = connparts
headers = self.read_headers(authenticate=True)
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n'%self.server_version) +
'\r\n'
)
self.wfile.flush()
self.establish_ssl(client_conn, host, port)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
if self.proxy_connect_state:
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
host, port, _ = self.proxy_connect_state
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, "https", method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
else:
r = http.parse_init_proxy(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, scheme, host, port, path, httpversion = r
headers = self.read_headers(authenticate=True)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def _read_request_transparent(self, client_conn, scheme, host, port):
"""
Read a transparent HTTP request. Transparent means that the client isn't aware of proxying.
In other words, the client request starts with
"GET /foo.html HTTP/1.1"
rather than
"CONNECT example.com:80 HTTP/1.1"
"""
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn,httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def create_http_request(flowheader, reqbuf):
sfp = StringIO(reqbuf)
method, url, httpversion = http.parse_init_http(sfp.readline())
host, port, path = parse_url(url)
headers = http.read_headers(sfp)
if not host:
if not headers.get("host"):
host = flowheader.dstip
else:
host = headers.get("host")[0]
if ":" in host:
host = string.rsplit(host, ":", maxsplit=1)[0]
if port == None:
port = flowheader.dport
# TODO: passing None as the second arg will produce and error if "expect" is in the headers
content = http.read_http_body_request(sfp, None, headers, httpversion, None)
# content = http.read_http_body(sfp, headers, True, None)
return flow.Request(
None,
httpversion,
host,
port,
"http",
method,
path,
headers,
content,
flowheader.ts_request_start,
flowheader.ts_request_finish,
)
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
if headers is None:
raise ProxyError(400, "Invalid headers")
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
def read_request(self, client_conn):
self.rfile.reset_timestamps()
if self.config.transparent_proxy:
orig = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not orig:
raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
host, port = orig
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content, self.rfile.first_byte_timestamp, utils.timestamp())
def test_read_http_body_request():
h = odict.ODictCaseless()
h["expect"] = ["100-continue"]
r = cStringIO.StringIO("testing")
w = cStringIO.StringIO()
assert http.read_http_body_request(r, w, h, (1, 1), None) == ""
assert "100 Continue" in w.getvalue()
def test_read_http_body_request():
h = odict.ODictCaseless()
h["expect"] = ["100-continue"]
r = cStringIO.StringIO("testing")
w = cStringIO.StringIO()
assert http.read_http_body_request(r, w, h, (1, 1), None) == ""
assert "100 Continue" in w.getvalue()
def read_request_transparent(self, client_conn):
orig = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not orig:
raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
self.log(client_conn, "transparent to %s:%s"%orig)
host, port = orig
if port in self.config.transparent_proxy["sslports"]:
scheme = "https"
if not self.ssl_established:
self.establish_ssl(client_conn, host, port)
else:
scheme = "http"
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn,httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def _read_request_origin_form(self, client_conn, scheme, host, port):
"""
Read a HTTP request with regular (origin-form) request line.
An example origin-form request line would be:
GET /foo.html HTTP/1.1
The request destination is already known from one of the following sources:
1) transparent proxy: destination provided by platform resolver
2) reverse proxy: fixed destination
3) regular proxy: known from CONNECT command.
"""
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, scheme,
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def _read_request_origin_form(self, client_conn, scheme, host, port):
"""
Read a HTTP request with regular (origin-form) request line.
An example origin-form request line would be:
GET /foo.html HTTP/1.1
The request destination is already known from one of the following sources:
1) transparent proxy: destination provided by platform resolver
2) reverse proxy: fixed destination
3) regular proxy: known from CONNECT command.
"""
if scheme.lower() == "https" and not self.ssl_established:
self.establish_ssl(client_conn, host, port)
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def handle_request(self):
"""
Returns True if handling should continue.
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
return
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s"%repr(line)
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
def read_request_reverse(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, "http",
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def read_request_reverse(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, "http", method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def _read_request_absolute_form(self, client_conn, line):
"""
When making a request to a proxy (other than CONNECT or OPTIONS),
a client must send the target uri in absolute-form.
An example absolute-form request line would be:
GET http://www.example.com/foo.html HTTP/1.1
"""
r = http.parse_init_proxy(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, scheme, host, port, path, httpversion = r
headers = self.read_headers(authenticate=True)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, scheme,
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s"%repr(line)
self.info(s)
return False, dict(type = "error", msg = s)
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type = "error", msg = s)
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type = "error", msg = s)
def _read_request_absolute_form(self, client_conn, line):
"""
When making a request to a proxy (other than CONNECT or OPTIONS),
a client must send the target uri in absolute-form.
An example absolute-form request line would be:
GET http://www.example.com/foo.html HTTP/1.1
"""
r = http.parse_init_proxy(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, scheme, host, port, path, httpversion = r
headers = self.read_headers(authenticate=True)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(
client_conn, httpversion, host, port, scheme, method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp()
)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
request_log = dict(
path=path,
method=method,
headers=headers.lst,
httpversion=httpversion,
sni=self.sni,
remote_address=self.client_address,
)
try:
content = http.read_http_body_request(self.rfile, self.wfile,
headers, httpversion, None)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type="error", msg=s)
keyinfo = self.clientcert.keyinfo,
)
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
clientcert = clientcert
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type = "error", msg = s)
for i in self.server.anchors:
if i[0].match(path):
self.info("crafting anchor: %s"%path)
aresp = language.parse_response(self.server.request_settings, i[1])
return self.serve_crafted(aresp, request_log)
if not self.server.nocraft and path.startswith(self.server.craftanchor):
spec = urllib.unquote(path)[len(self.server.craftanchor):]
self.info("crafting spec: %s"%spec)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
elif self.config.reverse_proxy:
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
else:
line = self.get_line(self.rfile)
if line == "":
return None
if line.startswith("CONNECT"):
r = http.parse_init_connect(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
host, port, httpversion = r
# FIXME: Discard additional headers sent to the proxy. Should I expose
# these to users?
while 1:
d = self.rfile.readline()
or self.config.cacert,
handle_sni=sni)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s" % repr(line))
method, path, httpversion = r
headers = self.read_headers(authenticate=False)
content = http.read_http_body_request(self.rfile, self.wfile, headers,
httpversion,
self.config.body_size_limit)
return flow.Request(client_conn, httpversion, host, port, scheme,
method, path, headers, content,
self.rfile.first_byte_timestamp, utils.timestamp())
def read_request_proxy(self, client_conn):
line = self.get_line(self.rfile)
if line == "":
return None
if not self.proxy_connect_state:
connparts = http.parse_init_connect(line)
if connparts:
host, port, httpversion = connparts
headers = self.read_headers(authenticate=True)
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
elif self.config.reverse_proxy:
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
else:
line = self.get_line(self.rfile)
if line == "":
return None
if line.startswith("CONNECT"):
host, port, httpversion = http.parse_init_connect(line)
# FIXME: Discard additional headers sent to the proxy. Should I expose
# these to users?
while 1:
d = self.rfile.readline()
if d == '\r\n' or d == '\n':
break
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
certfile = self.find_cert(host, port, None)
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
if self.proxy_connect_state:
host, port, httpversion = self.proxy_connect_state
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
else:
method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
