def handle(self):
try:
request = http.http1.read_request(self.rfile)
assert websockets.check_handshake(request.headers)
response = http.Response(
http_version=b"HTTP/1.1",
status_code=101,
reason=http.status_codes.RESPONSES.get(101).encode(),
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
sec_websocket_extensions='permessage-deflate'
if "permessage-deflate" in request.headers.values()
else ''),
content=b'',
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
self.wfile.write(http.http1.assemble_response(response))
self.wfile.flush()
self.server.handle_websockets(self.rfile, self.wfile)
except:
traceback.print_exc()
def handle_101_switching_protocols(self, f):
"""
Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
"""
# Check for WebSockets handshake
is_websockets = (f and websockets.check_handshake(f.request.headers)
and websockets.check_handshake(f.response.headers))
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
"info")
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()
def handle_101_switching_protocols(self, f):
"""
Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
"""
# Check for WebSockets handshake
is_websockets = (
f and
websockets.check_handshake(f.request.headers) and
websockets.check_handshake(f.response.headers)
)
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
"info"
)
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()
def _setup_connection(self):
client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
request = http.Request(
"authority",
"CONNECT",
"",
"localhost",
self.server.server.address.port,
"",
"HTTP/1.1",
content=b'')
client.wfile.write(http.http1.assemble_request(request))
client.wfile.flush()
response = http.http1.read_response(client.rfile, request)
if self.ssl:
client.convert_to_ssl()
assert client.ssl_established
request = http.Request(
"relative",
"GET",
"http",
"localhost",
self.server.server.address.port,
"/ws",
"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'')
client.wfile.write(http.http1.assemble_request(request))
client.wfile.flush()
response = http.http1.read_response(client.rfile, request)
assert websockets.check_handshake(response.headers)
return client
def _setup_connection(self):
client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
request = http.Request(
"authority",
"CONNECT",
"",
"127.0.0.1",
self.server.server.address[1],
"",
"HTTP/1.1",
content=b'')
client.wfile.write(http.http1.assemble_request(request))
client.wfile.flush()
response = http.http1.read_response(client.rfile, request)
if self.ssl:
client.convert_to_ssl()
assert client.ssl_established
request = http.Request(
"relative",
"GET",
"http",
"127.0.0.1",
self.server.server.address[1],
"/ws",
"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'')
client.wfile.write(http.http1.assemble_request(request))
client.wfile.flush()
response = http.http1.read_response(client.rfile, request)
assert websockets.check_handshake(response.headers)
return client
def setup_connection(self, extension=False):
self.client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
self.client.connect()
request = http.Request(
"authority",
"CONNECT",
"",
"127.0.0.1",
self.server.server.address[1],
"",
"HTTP/1.1",
content=b'')
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
if self.ssl:
self.client.convert_to_tls()
assert self.client.tls_established
request = http.Request(
"relative",
"GET",
"http",
"127.0.0.1",
self.server.server.address[1],
"/ws",
"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
sec_websocket_extensions="permessage-deflate" if extension else ""
),
content=b'')
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
assert websockets.check_handshake(response.headers)
def setup_connection(self, extension=False):
self.client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
self.client.connect()
request = http.Request(
"authority",
"CONNECT",
"",
"127.0.0.1",
self.server.server.address[1],
"",
"HTTP/1.1",
content=b'')
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
if self.ssl:
self.client.convert_to_tls()
assert self.client.tls_established
request = http.Request(
"relative",
"GET",
"http",
"127.0.0.1",
self.server.server.address[1],
"/ws",
"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
sec_websocket_extensions="permessage-deflate" if extension else ""
),
content=b'')
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
assert websockets.check_handshake(response.headers)
def handle(self):
try:
request = http.http1.read_request(self.rfile)
assert websockets.check_handshake(request.headers)
response = http.Response(
"HTTP/1.1",
101,
reason=http.status_codes.RESPONSES.get(101),
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
)
self.wfile.write(http.http1.assemble_response(response))
self.wfile.flush()
self.server.handle_websockets(self.rfile, self.wfile)
except:
traceback.print_exc()
def setup_connection(self, extension=False):
self.client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
self.client.connect()
request = make_connect_request(
("127.0.0.1", self.server.server.address[1]))
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
if self.ssl:
self.client.convert_to_tls()
assert self.client.tls_established
request = http.Request(
host="127.0.0.1",
port=self.server.server.address[1],
method=b"GET",
scheme=b"http",
authority=b"",
path=b"/ws",
http_version=b"HTTP/1.1",
headers=http.Headers(connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
sec_websocket_extensions="permessage-deflate"
if extension else ""),
content=b'',
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
assert websockets.check_handshake(response.headers)
def handle(self):
try:
request = http.http1.read_request(self.rfile)
assert websockets.check_handshake(request.headers)
response = http.Response(
"HTTP/1.1",
101,
reason=http.status_codes.RESPONSES.get(101),
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
)
self.wfile.write(http.http1.assemble_response(response))
self.wfile.flush()
self.server.handle_websockets(self.rfile, self.wfile)
except:
traceback.print_exc()
def _process_flow(self, f):
try:
request = self.get_request_from_client(f)
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return False
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
)
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return False
f.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return False
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers["Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
f.request.headers["Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(f)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(
f.request.host,
f.request.port,
f.request.scheme
)
self.get_response_from_server(f)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
self.send_response_to_client(f)
if self.check_close_connection(f):
return False
# Handle 101 Switching Protocols
if f.response.status_code == 101:
self.handle_101_switching_protocols(f)
return False # should never be reached
# Upstream Proxy Mode: Handle CONNECT
if f.request.first_line_format == "authority" and f.response.status_code == 200:
self.handle_upstream_mode_connect(f.request.copy())
return False
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)
)
finally:
if f:
f.live = False
return True
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
f = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
try:
request = self.get_request_from_client(f)
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e))
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return
f.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
except (exceptions.ProtocolException,
exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers[
"Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
f.request.headers[
"Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(f)
try:
if websockets.check_handshake(
request.headers) and websockets.check_client_version(
request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(f.request.host,
f.request.port,
f.request.scheme)
self.get_response_from_server(f)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
self.send_response_to_client(f)
if self.check_close_connection(f):
return
# Handle 101 Switching Protocols
if f.response.status_code == 101:
return self.handle_101_switching_protocols(f)
# Upstream Proxy Mode: Handle CONNECT
if f.request.first_line_format == "authority" and f.response.status_code == 200:
self.handle_upstream_mode_connect(f.request.copy())
return
except (exceptions.ProtocolException,
exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e))
finally:
if f:
f.live = False
def test_check_handshake(self, input, expected):
h = http.Headers(input)
assert websockets.check_handshake(h) == expected
def _process_flow(self, f):
try:
try:
request = self.read_request_headers(f)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen
# before/between requests.
return False
f.request = request
if request.first_line_format == "authority":
# The standards are silent on what we should do with a CONNECT
# request body, so although it's not common, it's allowed.
f.request.data.content = b"".join(
self.read_request_body(f.request)
)
f.request.timestamp_end = time.time()
self.channel.ask("http_connect", f)
if self.mode is HTTPMode.regular:
return self.handle_regular_connect(f)
elif self.mode is HTTPMode.upstream:
return self.handle_upstream_connect(f)
else:
msg = "Unexpected CONNECT request."
self.send_error_response(400, msg)
raise exceptions.ProtocolException(msg)
validate_request_form(self.mode, request)
self.channel.ask("requestheaders", f)
# Re-validate request form in case the user has changed something.
validate_request_form(self.mode, request)
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2
# here.
self.send_response(http.expect_continue_response)
request.headers.pop("expect")
if f.request.stream:
f.request.data.content = None
else:
f.request.data.content = b"".join(self.read_request_body(request))
request.timestamp_end = time.time()
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
# Request may be malformed at this point, so we unset it.
f.request = None
f.error = flow.Error(str(e))
self.channel.ask("error", f)
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
)
self.log("request", "debug", [repr(request)])
# set first line format to relative in regular mode,
# see https://github.com/mitmproxy/mitmproxy/issues/1759
if self.mode is HTTPMode.regular and request.first_line_format == "absolute":
request.first_line_format = "relative"
# update host header in reverse proxy mode
if self.config.options.mode.startswith("reverse:") and not self.config.options.keep_host_header:
f.request.host_header = self.config.upstream_server.address[0]
# Determine .scheme, .host and .port attributes for inline scripts. For
# absolute-form requests, they are directly given in the request. For
# authority-form requests, we only need to determine the request
# scheme. For relative-form requests, we need to determine host and
# port as well.
if self.mode is HTTPMode.transparent:
# Setting request.host also updates the host header, which we want
# to preserve
host_header = f.request.host_header
f.request.host = self.__initial_server_address[0]
f.request.port = self.__initial_server_address[1]
f.request.host_header = host_header # set again as .host overwrites this.
f.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", f)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
f.metadata['websocket'] = True
# We only support RFC6455 with WebSocket version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(
f.request.host,
f.request.port,
f.request.scheme
)
def get_response():
self.send_request_headers(f.request)
if f.request.stream:
chunks = self.read_request_body(f.request)
if callable(f.request.stream):
chunks = f.request.stream(chunks)
self.send_request_body(f.request, chunks)
else:
self.send_request_body(f.request, [f.request.data.content])
f.response = self.read_response_headers()
try:
get_response()
except exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise exceptions.ProtocolException(
"First and only attempt to get response via HTTP2 failed."
)
elif f.request.stream:
# We may have already consumed some request chunks already,
# so all we can do is signal downstream that upstream closed the connection.
self.send_error_response(408, "Request Timeout")
f.error = flow.Error(repr(e))
self.channel.ask("error", f)
return False
self.disconnect()
self.connect()
get_response()
# call the appropriate script hook - this is an opportunity for
# an inline script to set f.stream = True
self.channel.ask("responseheaders", f)
if f.response.stream:
f.response.data.content = None
else:
f.response.data.content = b"".join(
self.read_response_body(f.request, f.response)
)
f.response.timestamp_end = time.time()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
f.server_conn = self.server_conn
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
if not f.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_response(f.response)
else:
# streaming:
# First send the headers and then transfer the response incrementally
self.send_response_headers(f.response)
chunks = self.read_response_body(
f.request,
f.response
)
if callable(f.response.stream):
chunks = f.response.stream(chunks)
self.send_response_body(f.response, chunks)
f.response.timestamp_end = time.time()
if self.check_close_connection(f):
return False
# Handle 101 Switching Protocols
if f.response.status_code == 101:
# Handle a successful HTTP 101 Switching Protocols Response,
# received after e.g. a WebSocket upgrade request.
# Check for WebSocket handshake
is_websocket = (
websockets.check_handshake(f.request.headers) and
websockets.check_handshake(f.response.headers)
)
if is_websocket and not self.config.options.websocket:
self.log(
"Client requested WebSocket connection, but the protocol is disabled.",
"info"
)
if is_websocket and self.config.options.websocket:
layer = WebSocketLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()
return False # should never be reached
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)
)
finally:
if f:
f.live = False
return True
def handle_http_request(self, logger):
"""
Returns a (handler, log) tuple.
handler: Handler for the next request, or None to disconnect
log: A dictionary, or None
"""
with logger.ctx() as lg:
try:
req = self.protocol.read_request(self.rfile)
except exceptions.HttpReadDisconnect:
return None, None
except exceptions.HttpException as s:
s = str(s)
lg(s)
return None, dict(type="error", msg=s)
if req.method == 'CONNECT':
return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg)
method = req.method
path = req.path
http_version = req.http_version
headers = req.headers
first_line_format = req.first_line_format
clientcert = None
if self.clientcert:
clientcert = dict(
cn=self.clientcert.cn,
subject=self.clientcert.subject,
serial=self.clientcert.serial,
notbefore=self.clientcert.notbefore.isoformat(),
notafter=self.clientcert.notafter.isoformat(),
keyinfo=self.clientcert.keyinfo,
)
retlog = dict(
type="crafted",
protocol="http",
request=dict(
path=path,
method=method,
headers=headers.fields,
http_version=http_version,
sni=self.sni,
remote_address=self.address,
clientcert=clientcert,
first_line_format=first_line_format
),
cipher=None,
)
if self.ssl_established:
retlog["cipher"] = self.get_current_cipher()
m = utils.MemBool()
valid_websocket_handshake = websockets.check_handshake(headers)
self.settings.websocket_key = websockets.get_client_key(headers)
# If this is a websocket initiation, we respond with a proper
# server response, unless over-ridden.
if valid_websocket_handshake:
anchor_gen = language.parse_pathod("ws")
else:
anchor_gen = None
for regex, spec in self.server.anchors:
if regex.match(path):
anchor_gen = language.parse_pathod(spec, self.use_http2)
break
else:
if m(path.startswith(self.server.craftanchor)):
spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
if spec:
try:
anchor_gen = language.parse_pathod(spec, self.use_http2)
except language.ParseException as v:
lg("Parse error: %s" % v.msg)
anchor_gen = iter([self.make_http_error_response(
"Parse Error",
"Error parsing response spec: %s\n" % (
v.msg + v.marked()
)
)])
else:
if self.use_http2:
anchor_gen = iter([self.make_http_error_response(
"Spec Error",
"HTTP/2 only supports request/response with the craft anchor point: %s" %
self.server.craftanchor
)])
if not anchor_gen:
anchor_gen = iter([self.make_http_error_response(
"Not found",
"No valid craft request found"
)])
spec = next(anchor_gen)
if self.use_http2 and isinstance(spec, language.http2.Response):
spec.stream_id = req.stream_id
lg("crafting spec: %s" % spec)
nexthandler, retlog["response"] = self.http_serve_crafted(
spec,
lg
)
if nexthandler and valid_websocket_handshake:
self.protocol = protocols.websockets.WebsocketsProtocol(self)
return self.protocol.handle_websocket, retlog
else:
return nexthandler, retlog
def test_check_handshake(self, input, expected):
h = http.Headers(input)
assert websockets.check_handshake(h) == expected
def __call__(self, f):
m = (
(isinstance(f, http.HTTPFlow) and f.request and net_websockets.check_handshake(f.request.headers))
or isinstance(f, websocket.WebSocketFlow)
)
return m
def _process_flow(self, f):
try:
try:
request = self.read_request_headers(f)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen
# before/between requests.
return False
f.request = request
if request.first_line_format == "authority":
# The standards are silent on what we should do with a CONNECT
# request body, so although it's not common, it's allowed.
f.request.data.content = b"".join(
self.read_request_body(f.request)
)
f.request.timestamp_end = time.time()
self.channel.ask("http_connect", f)
if self.mode is HTTPMode.regular:
return self.handle_regular_connect(f)
elif self.mode is HTTPMode.upstream:
return self.handle_upstream_connect(f)
else:
msg = "Unexpected CONNECT request."
self.send_error_response(400, msg)
raise exceptions.ProtocolException(msg)
self.channel.ask("requestheaders", f)
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2
# here.
self.send_response(http.expect_continue_response)
request.headers.pop("expect")
request.data.content = b"".join(self.read_request_body(request))
request.timestamp_end = time.time()
validate_request_form(self.mode, request)
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
)
self.log("request", "debug", [repr(request)])
# set first line format to relative in regular mode,
# see https://github.com/mitmproxy/mitmproxy/issues/1759
if self.mode is HTTPMode.regular and request.first_line_format == "absolute":
request.first_line_format = "relative"
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers["Host"] = self.config.upstream_server.address.host
# Determine .scheme, .host and .port attributes for inline scripts. For
# absolute-form requests, they are directly given in the request. For
# authority-form requests, we only need to determine the request
# scheme. For relative-form requests, we need to determine host and
# port as well.
if self.mode is HTTPMode.transparent:
# Setting request.host also updates the host header, which we want
# to preserve
host_header = f.request.headers.get("host", None)
f.request.host = self.__initial_server_conn.address.host
f.request.port = self.__initial_server_conn.address.port
if host_header:
f.request.headers["host"] = host_header
f.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", f)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(
f.request.host,
f.request.port,
f.request.scheme
)
def get_response():
self.send_request(f.request)
f.response = self.read_response_headers()
try:
get_response()
except exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise exceptions.ProtocolException(
"First and only attempt to get response via HTTP2 failed."
)
self.disconnect()
self.connect()
get_response()
# call the appropriate script hook - this is an opportunity for
# an inline script to set f.stream = True
self.channel.ask("responseheaders", f)
if f.response.stream:
f.response.data.content = None
else:
f.response.data.content = b"".join(
self.read_response_body(f.request, f.response)
)
f.response.timestamp_end = time.time()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
f.server_conn = self.server_conn
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
if not f.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_response(f.response)
else:
# streaming:
# First send the headers and then transfer the response incrementally
self.send_response_headers(f.response)
chunks = self.read_response_body(
f.request,
f.response
)
if callable(f.response.stream):
chunks = f.response.stream(chunks)
self.send_response_body(f.response, chunks)
f.response.timestamp_end = time.time()
if self.check_close_connection(f):
return False
# Handle 101 Switching Protocols
if f.response.status_code == 101:
# Handle a successful HTTP 101 Switching Protocols Response,
# received after e.g. a WebSocket upgrade request.
# Check for WebSockets handshake
is_websockets = (
websockets.check_handshake(f.request.headers) and
websockets.check_handshake(f.response.headers)
)
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is disabled.",
"info"
)
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()
return False # should never be reached
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)
)
finally:
if f:
f.live = False
return True
def handle_http_request(self, logger):
"""
Returns a (handler, log) tuple.
handler: Handler for the next request, or None to disconnect
log: A dictionary, or None
"""
with logger.ctx() as lg:
try:
req = self.protocol.read_request(self.rfile)
except exceptions.HttpReadDisconnect:
return None, None
except exceptions.HttpException as s:
s = str(s)
lg(s)
return None, dict(type="error", msg=s)
if req.method == 'CONNECT':
return self.protocol.handle_http_connect(
[req.host, req.port, req.http_version], lg)
method = req.method
path = req.path
http_version = req.http_version
headers = req.headers
first_line_format = req.first_line_format
clientcert = None
if self.clientcert:
clientcert = dict(
cn=self.clientcert.cn,
subject=self.clientcert.subject,
serial=self.clientcert.serial,
notbefore=self.clientcert.notbefore.isoformat(),
notafter=self.clientcert.notafter.isoformat(),
keyinfo=self.clientcert.keyinfo,
)
retlog = dict(
type="crafted",
protocol="http",
request=dict(path=path,
method=method,
headers=headers.fields,
http_version=http_version,
sni=self.sni,
remote_address=self.address,
clientcert=clientcert,
first_line_format=first_line_format),
cipher=None,
)
if self.tls_established:
retlog["cipher"] = self.get_current_cipher()
m = utils.MemBool()
valid_websocket_handshake = websockets.check_handshake(headers)
self.settings.websocket_key = websockets.get_client_key(headers)
# If this is a websocket initiation, we respond with a proper
# server response, unless over-ridden.
if valid_websocket_handshake:
anchor_gen = language.parse_pathod("ws")
else:
anchor_gen = None
for regex, spec in self.server.anchors:
if regex.match(path):
anchor_gen = language.parse_pathod(spec, self.use_http2)
break
else:
if m(path.startswith(self.server.craftanchor)):
spec = urllib.parse.unquote(
path)[len(self.server.craftanchor):]
if spec:
try:
anchor_gen = language.parse_pathod(
spec, self.use_http2)
except language.ParseException as v:
lg("Parse error: %s" % v.msg)
anchor_gen = iter([
self.make_http_error_response(
"Parse Error",
"Error parsing response spec: %s\n" %
(v.msg + v.marked()))
])
else:
if self.use_http2:
anchor_gen = iter([
self.make_http_error_response(
"Spec Error",
"HTTP/2 only supports request/response with the craft anchor point: %s"
% self.server.craftanchor)
])
if not anchor_gen:
anchor_gen = iter([
self.make_http_error_response(
"Not found", "No valid craft request found")
])
spec = next(anchor_gen)
if self.use_http2 and isinstance(spec, language.http2.Response):
spec.stream_id = req.stream_id
lg("crafting spec: %s" % spec)
nexthandler, retlog["response"] = self.http_serve_crafted(spec, lg)
if nexthandler and valid_websocket_handshake:
self.protocol = protocols.websockets.WebsocketsProtocol(self)
return self.protocol.handle_websocket, retlog
else:
return nexthandler, retlog
