def http_connect(self, connect_to):
self.wfile.write(
'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
'\r\n'
)
self.wfile.flush()
l = self.rfile.readline()
if not l:
raise PathocError("Proxy CONNECT failed")
parsed = http.parse_response_line(l)
if not parsed[1] == 200:
raise PathocError("Proxy CONNECT failed: %s - %s"%(parsed[1], parsed[2]))
http.read_headers(self.rfile)
def create_http_request(flowheader, reqbuf):
sfp = StringIO(reqbuf)
method, url, httpversion = http.parse_init_http(sfp.readline())
host, port, path = parse_url(url)
headers = http.read_headers(sfp)
if not host:
if not headers.get("host"):
host = flowheader.dstip
else:
host = headers.get("host")[0]
if ":" in host:
host = string.rsplit(host, ":", maxsplit=1)[0]
if port == None:
port = flowheader.dport
# TODO: passing None as the second arg will produce and error if "expect" is in the headers
content = http.read_http_body_request(sfp, None, headers, httpversion, None)
# content = http.read_http_body(sfp, headers, True, None)
return flow.Request(
None,
httpversion,
host,
port,
"http",
method,
path,
headers,
content,
flowheader.ts_request_start,
flowheader.ts_request_finish,
)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
m = utils.MemBool()
if m(http.parse_init_connect(line)):
headers = http.read_headers(self.rfile)
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
if not self.server.ssloptions.not_after_connect:
try:
self.convert_to_ssl(
self.server.ssloptions.certfile,
self.server.ssloptions.keyfile,
)
except tcp.NetLibError, v:
s = str(v)
self.info(s)
return False, dict(type = "error", msg = s)
return True, None
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
try:
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
except tcp.NetLibError, v:
raise ProxyError(400, str(v))
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
def http_connect(self, connect_to, wfile, rfile):
wfile.write(
'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
'\r\n'
)
wfile.flush()
rfile.readline()
headers = http.read_headers(self.rfile)
def from_stream(cls, rfile, include_content=True, body_size_limit=None):
"""
Parse an HTTP request from a file stream
"""
httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
= None, None, None, None, None, None, None, None, None, None
if hasattr(rfile, "reset_timestamps"):
rfile.reset_timestamps()
request_line = get_line(rfile)
if hasattr(rfile, "first_byte_timestamp"):
timestamp_start = rfile.first_byte_timestamp
else:
timestamp_start = utils.timestamp()
request_line_parts = http.parse_init(request_line)
if not request_line_parts:
raise http.HttpError(
400, "Bad HTTP request line: %s" % repr(request_line))
method, path, httpversion = request_line_parts
if path == '*':
form_in = "asterisk"
elif path.startswith("/"):
form_in = "origin"
if not netlib.utils.isascii(path):
raise http.HttpError(
400, "Bad HTTP request line: %s" % repr(request_line))
elif method.upper() == 'CONNECT':
form_in = "authority"
r = http.parse_init_connect(request_line)
if not r:
raise http.HttpError(
400, "Bad HTTP request line: %s" % repr(request_line))
host, port, _ = r
path = None
else:
form_in = "absolute"
r = http.parse_init_proxy(request_line)
if not r:
raise http.HttpError(
400, "Bad HTTP request line: %s" % repr(request_line))
_, scheme, host, port, path, _ = r
headers = http.read_headers(rfile)
if headers is None:
raise http.HttpError(400, "Invalid headers")
if include_content:
content = http.read_http_body(rfile, headers, body_size_limit,
True)
timestamp_end = utils.timestamp()
return HTTPRequest(form_in, method, scheme, host, port, path,
httpversion, headers, content, timestamp_start,
timestamp_end)
def handle_request(self):
"""
Returns True if handling should continue.
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
return
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s"%repr(line)
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
self.server.add_log(
dict(
type = "error",
msg = s
)
)
return
def test_read_multi(self):
data = """
Header: one
Header: two
\r\n
"""
data = textwrap.dedent(data)
data = data.strip()
s = cStringIO.StringIO(data)
h = http.read_headers(s)
assert h.lst == [["Header", "one"], ["Header", "two"]]
def test_read_continued(self):
data = """
Header: one
\ttwo
Header2: three
\r\n
"""
data = textwrap.dedent(data)
data = data.strip()
s = cStringIO.StringIO(data)
h = http.read_headers(s)
assert h.lst == [["Header", "one\r\n two"], ["Header2", "three"]]
def read_headers(self, authenticate=False):
headers = http.read_headers(self.rfile)
if headers is None:
raise ProxyError(400, "Invalid headers")
if authenticate and self.config.authenticator:
if self.config.authenticator.authenticate(headers):
self.config.authenticator.clean(headers)
else:
raise ProxyError(
407, "Proxy Authentication Required",
self.config.authenticator.auth_challenge_headers())
return headers
def from_stream(cls, rfile, include_content=True, body_size_limit=None):
"""
Parse an HTTP request from a file stream
"""
httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \
= None, None, None, None, None, None, None, None, None, None
if hasattr(rfile, "reset_timestamps"):
rfile.reset_timestamps()
request_line = get_line(rfile)
if hasattr(rfile, "first_byte_timestamp"):
timestamp_start = rfile.first_byte_timestamp
else:
timestamp_start = utils.timestamp()
request_line_parts = http.parse_init(request_line)
if not request_line_parts:
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
method, path, httpversion = request_line_parts
if path == '*':
form_in = "asterisk"
elif path.startswith("/"):
form_in = "origin"
if not netlib.utils.isascii(path):
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
elif method.upper() == 'CONNECT':
form_in = "authority"
r = http.parse_init_connect(request_line)
if not r:
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
host, port, _ = r
path = None
else:
form_in = "absolute"
r = http.parse_init_proxy(request_line)
if not r:
raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line))
_, scheme, host, port, path, _ = r
headers = http.read_headers(rfile)
if headers is None:
raise http.HttpError(400, "Invalid headers")
if include_content:
content = http.read_http_body(rfile, headers, body_size_limit, True)
timestamp_end = utils.timestamp()
return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content,
timestamp_start, timestamp_end)
def read_headers(self, authenticate=False):
headers = http.read_headers(self.rfile)
if headers is None:
raise ProxyError(400, "Invalid headers")
if authenticate and self.config.authenticator:
if self.config.authenticator.authenticate(headers):
self.config.authenticator.clean(headers)
else:
raise ProxyError(
407,
"Proxy Authentication Required",
self.config.authenticator.auth_challenge_headers()
)
return headers
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
if self.server.logreq:
self.rfile.start_log()
if self.server.logresp:
self.wfile.start_log()
line = http.get_request_line(self.rfile)
if not line:
# Normal termination
return False
m = utils.MemBool()
if m(http.parse_init_connect(line)):
headers = http.read_headers(self.rfile)
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
if not self.server.ssloptions.not_after_connect:
try:
cert, key, chain_file = self.server.ssloptions.get_cert(
m.v[0]
)
self.convert_to_ssl(
cert, key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
)
except tcp.NetLibError, v:
s = str(v)
self.info(s)
self.addlog(dict(type="error", msg=s))
return False
return True
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s"%repr(line)
self.info(s)
return False, dict(type = "error", msg = s)
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type = "error", msg = s)
request_log = dict(
path = path,
method = method,
headers = headers.lst,
httpversion = httpversion,
sni = self.sni,
remote_address = self.client_address,
)
try:
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, None
)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type = "error", msg = s)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
parts = http.parse_init_http(line)
if not parts:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
method, path, httpversion = parts
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
request_log = dict(
path=path,
method=method,
headers=headers.lst,
httpversion=httpversion,
sni=self.sni,
remote_address=self.client_address,
)
try:
content = http.read_http_body_request(self.rfile, self.wfile,
headers, httpversion, None)
except http.HttpError, s:
s = str(s)
self.info(s)
return False, dict(type="error", msg=s)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
if self.server.logreq:
self.rfile.start_log()
if self.server.logresp:
self.wfile.start_log()
line = http.get_request_line(self.rfile)
if not line:
# Normal termination
return False
m = utils.MemBool()
if m(http.parse_init_connect(line)):
headers = http.read_headers(self.rfile)
self.wfile.write('HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
'\r\n')
self.wfile.flush()
if not self.server.ssloptions.not_after_connect:
try:
cert, key, chain_file = self.server.ssloptions.get_cert(
m.v[0])
self.convert_to_ssl(
cert,
key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.
request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
)
except tcp.NetLibError, v:
s = str(v)
self.info(s)
self.addlog(dict(type="error", msg=s))
return False
return True
def read_request(self, client_conn):
if self.config.transparent_proxy:
host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
scheme = "https"
certfile = self.find_cert(host, port, None)
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
else:
scheme = "http"
host = self.sni or host
line = self.get_line(self.rfile)
if line == "":
return None
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
elif self.config.reverse_proxy:
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
else:
line = self.get_line(self.rfile)
if line == "":
return None
if line.startswith("CONNECT"):
host, port, httpversion = http.parse_init_connect(line)
# FIXME: Discard additional headers sent to the proxy. Should I expose
# these to users?
while 1:
d = self.rfile.readline()
if d == '\r\n' or d == '\n':
break
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n'%version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
certfile = self.find_cert(host, port, None)
self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert)
self.proxy_connect_state = (host, port, httpversion)
line = self.rfile.readline(line)
if self.proxy_connect_state:
host, port, httpversion = self.proxy_connect_state
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%line)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "https", method, path, headers, content)
else:
method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, scheme, method, path, headers, content)
elif self.config.reverse_proxy:
line = self.get_line(self.rfile)
if line == "":
return None
scheme, host, port = self.config.reverse_proxy
r = http.parse_init_http(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
method, path, httpversion = r
headers = http.read_headers(self.rfile)
content = http.read_http_body_request(
self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
)
return flow.Request(client_conn, httpversion, host, port, "http", method, path, headers, content)
else:
line = self.get_line(self.rfile)
if line == "":
return None
if line.startswith("CONNECT"):
r = http.parse_init_connect(line)
if not r:
raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
host, port, httpversion = r
# FIXME: Discard additional headers sent to the proxy. Should I expose
# these to users?
def _read(self, data, verbatim=False):
if not verbatim:
data = textwrap.dedent(data)
data = data.strip()
s = cStringIO.StringIO(data)
return http.read_headers(s)
)
except tcp.NetLibError, v:
s = str(v)
self.info(s)
return False, dict(type="error", msg=s)
return True, None
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
clientcert = None
if self.clientcert:
clientcert = dict(
cn=self.clientcert.cn,
subject=self.clientcert.subject,
serial=self.clientcert.serial,
notbefore=self.clientcert.notbefore.isoformat(),
notafter=self.clientcert.notafter.isoformat(),
keyinfo=self.clientcert.keyinfo,
)
def _read(self, data, verbatim=False):
if not verbatim:
data = textwrap.dedent(data)
data = data.strip()
s = cStringIO.StringIO(data)
return http.read_headers(s)
def handle_request(self):
"""
Returns a (again, log) tuple.
again: True if request handling should continue.
log: A dictionary, or None
"""
line = self.rfile.readline()
if line == "\r\n" or line == "\n": # Possible leftover from previous message
line = self.rfile.readline()
if line == "":
# Normal termination
return False, None
m = utils.MemBool()
if m(http.parse_init_connect(line)):
headers = http.read_headers(self.rfile)
self.wfile.write(
'HTTP/1.1 200 Connection established\r\n' +
('Proxy-agent: %s\r\n' % version.NAMEVERSION) +
'\r\n'
)
self.wfile.flush()
if not self.server.ssloptions.not_after_connect:
try:
cert, key, chain_file = self.server.ssloptions.get_cert(m.v[0])
self.convert_to_ssl(
cert, key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
)
except tcp.NetLibError as v:
s = str(v)
self.info(s)
return False, dict(type="error", msg=s)
return True, None
elif m(http.parse_init_proxy(line)):
method, _, _, _, path, httpversion = m.v
elif m(http.parse_init_http(line)):
method, path, httpversion = m.v
else:
s = "Invalid first line: %s" % repr(line)
self.info(s)
return False, dict(type="error", msg=s)
headers = http.read_headers(self.rfile)
if headers is None:
s = "Invalid headers"
self.info(s)
return False, dict(type="error", msg=s)
clientcert = None
if self.clientcert:
clientcert = dict(
cn=self.clientcert.cn,
subject=self.clientcert.subject,
serial=self.clientcert.serial,
notbefore=self.clientcert.notbefore.isoformat(),
notafter=self.clientcert.notafter.isoformat(),
keyinfo=self.clientcert.keyinfo,
)
retlog = dict(
type="crafted",
request=dict(
path=path,
method=method,
headers=headers.lst,
httpversion=httpversion,
sni=self.sni,
remote_address=self.address(),
clientcert=clientcert,
),
cipher=None,
)
if self.ssl_established:
retlog["cipher"] = self.get_current_cipher()
try:
content = http.read_http_body(
self.rfile, headers, None,
method, None, True
)
except http.HttpError as s:
s = str(s)
self.info(s)
return False, dict(type="error", msg=s)
for i in self.server.anchors:
if i[0].match(path):
self.info("crafting anchor: %s" % path)
again, retlog["response"] = self.serve_crafted(i[1])
return again, retlog
if not self.server.nocraft and path.startswith(self.server.craftanchor):
spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
self.info("crafting spec: %s" % spec)
try:
crafted = language.parse_response(spec)
except language.ParseException as v:
self.info("Parse error: %s" % v.msg)
crafted = language.make_error_response(
"Parse Error",
"Error parsing response spec: %s\n" % v.msg + v.marked()
)
again, retlog["response"] = self.serve_crafted(crafted)
return again, retlog
elif self.server.noweb:
crafted = language.make_error_response("Access Denied")
language.serve(crafted, self.wfile, self.server.request_settings)
return False, dict(
type="error",
msg="Access denied: web interface disabled"
)
else:
self.info("app: %s %s" % (method, path))
req = wsgi.Request("http", method, path, headers, content)
flow = wsgi.Flow(self.address, req)
sn = self.connection.getsockname()
a = wsgi.WSGIAdaptor(
self.server.app,
sn[0],
self.server.address.port,
version.NAMEVERSION
)
a.serve(flow, self.wfile)
return True, None