def handle_regular_connect(self, f):
self.connect_request = True
try:
self.set_server((f.request.host, f.request.port))
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution
# and connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
if f.response:
resp = f.response
else:
resp = http.make_connect_response(f.request.data.http_version)
self.send_response(resp)
if is_ok(resp.status_code):
layer = self.ctx.next_layer(self)
layer()
return False
def check_killed(self, emit_error_hook: bool) -> layer.CommandGenerator[bool]:
killed_by_us = (
self.flow.error and self.flow.error.msg == flow.Error.KILLED_MESSAGE
)
# The client may have closed the connection while we were waiting for the hook to complete.
# We peek into the event queue to see if that is the case.
killed_by_remote = None
for evt in self._paused_event_queue:
if isinstance(evt, RequestProtocolError):
killed_by_remote = evt.message
break
if killed_by_remote:
if not self.flow.error:
self.flow.error = flow.Error(killed_by_remote)
if killed_by_us or killed_by_remote:
if emit_error_hook:
yield HttpErrorHook(self.flow)
# Use the special NO_RESPONSE status code to make sure that no error message is sent to the client.
yield SendHttp(
ResponseProtocolError(self.stream_id, "killed", status_codes.NO_RESPONSE),
self.context.client
)
self._handle_event = self.state_errored
return True
return False
def handle_protocol_error(
self,
event: Union[RequestProtocolError, ResponseProtocolError]
) -> layer.CommandGenerator[None]:
is_client_error_but_we_already_talk_upstream = (
isinstance(event, RequestProtocolError)
and self.client_state in (self.state_stream_request_body, self.state_done)
and self.server_state != self.state_errored
)
need_error_hook = not (
self.client_state in (self.state_wait_for_request_headers, self.state_errored)
or
self.server_state in (self.state_done, self.state_errored)
)
if is_client_error_but_we_already_talk_upstream:
yield SendHttp(event, self.context.server)
self.client_state = self.state_errored
if need_error_hook:
# We don't want to trigger both a response hook and an error hook,
# so we need to check if the response is done yet or not.
self.flow.error = flow.Error(event.message)
yield HttpErrorHook(self.flow)
if (yield from self.check_killed(False)):
return
if isinstance(event, ResponseProtocolError):
if self.client_state != self.state_errored:
yield SendHttp(event, self.context.client)
self.server_state = self.state_errored
def check_killed(self,
emit_error_hook: bool) -> layer.CommandGenerator[bool]:
killed_by_us = (self.flow.error
and self.flow.error.msg == flow.Error.KILLED_MESSAGE)
# The client may have closed the connection while we were waiting for the hook to complete.
# We peek into the event queue to see if that is the case.
killed_by_remote = None
for evt in self._paused_event_queue:
if isinstance(evt, RequestProtocolError):
killed_by_remote = evt.message
break
if killed_by_remote:
if not self.flow.error:
self.flow.error = flow.Error(killed_by_remote)
if killed_by_us or killed_by_remote:
if emit_error_hook:
yield HttpErrorHook(self.flow)
# For HTTP/2 we only want to kill the specific stream, for HTTP/1 we want to kill the connection
# *without* sending an HTTP response (that could be achieved by the user by setting flow.response).
if self.context.client.alpn == b"h2":
yield SendHttp(ResponseProtocolError(self.stream_id, "killed"),
self.context.client)
else:
if self.context.client.state & ConnectionState.CAN_WRITE:
yield commands.CloseConnection(self.context.client)
self._handle_event = self.state_errored
return True
return False
def __call__(self):
self.connect()
if not self.ignore:
f = tcp.TCPFlow(self.client_conn, self.server_conn, self)
self.channel.ask("tcp_start", f)
buf = memoryview(bytearray(self.chunk_size))
client = self.client_conn.connection
server = self.server_conn.connection
conns = [client, server]
# https://github.com/openssl/openssl/issues/6234
for conn in conns:
if isinstance(conn, SSL.Connection) and hasattr(
SSL._lib, "SSL_clear_mode"):
SSL._lib.SSL_clear_mode(conn._ssl,
SSL._lib.SSL_MODE_AUTO_RETRY)
try:
while not self.channel.should_exit.is_set():
r = mitmproxy.net.tcp.ssl_read_select(conns, 10)
for conn in r:
dst = server if conn == client else client
try:
size = conn.recv_into(buf, self.chunk_size)
except (SSL.WantReadError, SSL.WantWriteError):
continue
if not size:
conns.remove(conn)
# Shutdown connection to the other peer
if isinstance(conn, SSL.Connection):
# We can't half-close a connection, so we just close everything here.
# Sockets will be cleaned up on a higher level.
return
else:
dst.shutdown(socket.SHUT_WR)
if len(conns) == 0:
return
continue
tcp_message = tcp.TCPMessage(dst == server,
buf[:size].tobytes())
if not self.ignore:
f.messages.append(tcp_message)
self.channel.ask("tcp_message", f)
dst.sendall(tcp_message.content)
except (OSError, exceptions.TcpException, SSL.Error) as e:
if not self.ignore:
f.error = flow.Error(
"TCP connection closed unexpectedly: {}".format(repr(e)))
self.channel.tell("tcp_error", f)
finally:
if not self.ignore:
self.channel.tell("tcp_end", f)
def test_getset_state(self):
f = tutils.tflow(resp=True)
state = f.get_state()
assert f.get_state() == http.HTTPFlow.from_state(state).get_state()
f.response = None
f.error = flow.Error("error")
state = f.get_state()
assert f.get_state() == http.HTTPFlow.from_state(state).get_state()
f2 = f.copy()
f2.id = f.id # copy creates a different uuid
assert f.get_state() == f2.get_state()
assert not f == f2
f2.error = flow.Error("e2")
assert not f == f2
f.set_state(f2.get_state())
assert f.get_state() == f2.get_state()
def test_getset_state(self):
f = tflow(resp=True)
state = f.get_state()
assert f.get_state() == HTTPFlow.from_state(state).get_state()
f.response = None
f.error = flow.Error("error")
state = f.get_state()
assert f.get_state() == HTTPFlow.from_state(state).get_state()
f2 = f.copy()
f2.id = f.id # copy creates a different uuid
assert f.get_state() == f2.get_state()
assert not f == f2
f2.error = flow.Error("e2")
assert not f == f2
f2.backup()
f2.intercept() # to change the state
f.set_state(f2.get_state())
assert f.get_state() == f2.get_state()
def __call__(self):
self.flow = WebSocketFlow(self.client_conn, self.server_conn,
self.handshake_flow)
self.flow.metadata['websocket_handshake'] = self.handshake_flow.id
self.handshake_flow.metadata['websocket_flow'] = self.flow.id
self.channel.ask("websocket_start", self.flow)
conns = [c.connection for c in self.connections.keys()]
close_received = False
try:
while not self.channel.should_exit.is_set():
self._inject_messages(self.client_conn,
self.flow._inject_messages_client)
self._inject_messages(self.server_conn,
self.flow._inject_messages_server)
r = tcp.ssl_read_select(conns, 0.1)
for conn in r:
source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn
other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn
is_server = (source_conn == self.server_conn)
header, frame, consumed_bytes = websocket.read_frame(
source_conn.rfile)
self.log(
"WebSocket Frame from {}: {}, {}".format(
"server" if is_server else "client",
header,
frame,
), "debug")
data = self.connections[source_conn].receive_data(
consumed_bytes)
source_conn.send(data)
if close_received:
return
for event in self.connections[source_conn].events():
if not self._handle_event(event, source_conn,
other_conn, is_server):
if not close_received:
close_received = True
except (socket.error, exceptions.TcpException, SSL.Error) as e:
s = 'server' if is_server else 'client'
self.flow.error = flow.Error(
"WebSocket connection closed unexpectedly by {}: {}".format(
s, repr(e)))
self.channel.tell("websocket_error", self.flow)
finally:
self.flow.ended = True
self.channel.tell("websocket_end", self.flow)
def websocket_end(self, f: http.HTTPFlow):
assert f.websocket is not None # satisfy type checker
if self.match(f):
if f.websocket.close_code in {1000, 1001, 1005}:
c = 'client' if f.websocket.closed_by_client else 'server'
self.echo(f"WebSocket connection closed by {c}: {f.websocket.close_code} {f.websocket.close_reason}")
else:
error = flow.Error(f"WebSocket Error: {self.format_websocket_error(f.websocket)}")
self.echo(
f"Error in WebSocket connection to {human.format_address(f.server_conn.address)}: {error}",
fg="red"
)
def start(self, _) -> layer.CommandGenerator[None]:
if self.flow:
yield TcpStartHook(self.flow)
if not self.context.server.connected:
err = yield commands.OpenConnection(self.context.server)
if err:
if self.flow:
self.flow.error = flow.Error(str(err))
yield TcpErrorHook(self.flow)
yield commands.CloseConnection(self.context.client)
self._handle_event = self.done
return
self._handle_event = self.relay_messages
def test_err(self):
c = state.State()
f = tutils.tflow()
c.add_flow(f)
f.error = flow.Error("message")
assert c.update_flow(f)
c = state.State()
f = tutils.tflow()
c.add_flow(f)
c.set_view_filter("~e")
assert not c.view
f.error = tutils.terr()
assert c.update_flow(f)
assert c.view
def __call__(self):
self.connect()
if not self.ignore:
f = udp.UDPFlow(self.client_conn, self.server_conn, self)
self.channel.ask("udp_start", f)
buf = memoryview(bytearray(self.chunk_size))
client = self.client_conn.connection
server = self.server_conn.connection
conns = [client, server]
# https://github.com/openssl/openssl/issues/6234
#for conn in conns:
# if isinstance(conn, SSL.Connection) and hasattr(SSL._lib, "SSL_clear_mode"):
# SSL._lib.SSL_clear_mode(conn._ssl, SSL._lib.SSL_MODE_AUTO_RETRY)
try:
while not self.channel.should_exit.is_set():
r = mitmproxy.net.udp.read_select(conns, 10)
for conn in r:
dst = server if conn == client else client
size = conn.recv_into(buf, self.chunk_size)
if not size:
conns.remove(conn)
# Shutdown connection to the other peer
dst.shutdown(socket.SHUT_WR)
if len(conns) == 0:
return
continue
udp_message = udp.UDPMessage(dst == server,
buf[:size].tobytes())
if not self.ignore:
f.messages.append(udp_message)
self.channel.ask("udp_message", f)
dst.sendall(udp_message.content)
except (OSError, exceptions.UdpException) as e:
if not self.ignore:
f.error = flow.Error(
"UDP connection closed unexpectedly: {}".format(repr(e)))
self.channel.tell("udp_error", f)
finally:
if not self.ignore:
self.channel.tell("udp_end", f)
async def test_all(self):
opts = options.Options(
mode="reverse:https://use-this-domain"
)
s = State()
with taddons.context(s, options=opts) as ctx:
f = tflow.tflow(req=None)
await ctx.master.addons.handle_lifecycle(server_hooks.ClientConnectedHook(f.client_conn))
f.request = mitmproxy.test.tutils.treq()
await ctx.master.addons.handle_lifecycle(layers.http.HttpRequestHook(f))
assert len(s.flows) == 1
f.response = mitmproxy.test.tutils.tresp()
await ctx.master.addons.handle_lifecycle(layers.http.HttpResponseHook(f))
assert len(s.flows) == 1
await ctx.master.addons.handle_lifecycle(server_hooks.ClientDisconnectedHook(f.client_conn))
f.error = flow.Error("msg")
await ctx.master.addons.handle_lifecycle(layers.http.HttpErrorHook(f))
def test_all(self):
s = tservers.TestState()
fm = master.Master(None, DummyServer())
fm.addons.add(s)
f = tflow.tflow(req=None)
fm.clientconnect(f.client_conn)
f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
fm.request(f)
assert s.flow_count() == 1
f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
fm.response(f)
assert s.flow_count() == 1
fm.clientdisconnect(f.client_conn)
f.error = flow.Error("msg")
fm.error(f)
fm.shutdown()
def test_all(self):
s = tservers.TestState()
fm = master.Master(None, DummyServer())
fm.addons.add(s)
f = tflow.tflow(req=None)
fm.addons.handle_lifecycle("clientconnect", f.client_conn)
f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
fm.addons.handle_lifecycle("request", f)
assert len(s.flows) == 1
f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
fm.addons.handle_lifecycle("response", f)
assert len(s.flows) == 1
fm.addons.handle_lifecycle("clientdisconnect", f.client_conn)
f.error = flow.Error("msg")
fm.addons.handle_lifecycle("error", f)
fm.shutdown()
async def test_all(self):
opts = options.Options(
mode="reverse:https://use-this-domain"
)
s = tservers.TestState()
with taddons.context(s, options=opts) as ctx:
f = tflow.tflow(req=None)
await ctx.master.addons.handle_lifecycle("clientconnect", f.client_conn)
f.request = mitmproxy.test.tutils.treq()
await ctx.master.addons.handle_lifecycle("request", f)
assert len(s.flows) == 1
f.response = mitmproxy.test.tutils.tresp()
await ctx.master.addons.handle_lifecycle("response", f)
assert len(s.flows) == 1
await ctx.master.addons.handle_lifecycle("clientdisconnect", f.client_conn)
f.error = flow.Error("msg")
await ctx.master.addons.handle_lifecycle("error", f)
def __call__(self):
self.flow = WebSocketFlow(self.client_conn, self.server_conn,
self.handshake_flow, self)
self.flow.metadata['websocket_handshake'] = self.handshake_flow.id
self.handshake_flow.metadata['websocket_flow'] = self.flow.id
self.channel.ask("websocket_start", self.flow)
conns = [c.connection for c in self.connections.keys()]
close_received = False
try:
while not self.channel.should_exit.is_set():
r = tcp.ssl_read_select(conns, 0.1)
for conn in r:
source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn
other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn
is_server = (source_conn == self.server_conn)
frame = websockets.Frame.from_file(source_conn.rfile)
self.connections[source_conn].receive_bytes(bytes(frame))
source_conn.send(
self.connections[source_conn].bytes_to_send())
if close_received:
return
for event in self.connections[source_conn].events():
if not self._handle_event(event, source_conn,
other_conn, is_server):
if not close_received:
close_received = True
except (socket.error, exceptions.TcpException, SSL.Error) as e:
s = 'server' if is_server else 'client'
self.flow.error = flow.Error(
"WebSocket connection closed unexpectedly by {}: {}".format(
s, repr(e)))
self.channel.tell("websocket_error", self.flow)
finally:
self.channel.tell("websocket_end", self.flow)
def test_all(self):
s = tservers.TestState()
fm = master.Master(None)
fm.addons.add(s)
f = tflow.tflow(req=None)
fm.addons.handle_lifecycle("clientconnect", f.client_conn)
f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq())
fm.addons.handle_lifecycle("request", f)
assert len(s.flows) == 1
f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp())
fm.addons.handle_lifecycle("response", f)
assert len(s.flows) == 1
fm.addons.handle_lifecycle("clientdisconnect", f.client_conn)
f.error = flow.Error("msg")
fm.addons.handle_lifecycle("error", f)
fm.tell("foo", f)
with pytest.raises(ControlException):
fm.tick(timeout=1)
fm.shutdown()
def _process_flow(self, f):
try:
try:
request = self.read_request_headers(f)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen
# before/between requests.
return False
f.request = request
if request.first_line_format == "authority":
# The standards are silent on what we should do with a CONNECT
# request body, so although it's not common, it's allowed.
f.request.data.content = b"".join(
self.read_request_body(f.request)
)
f.request.timestamp_end = time.time()
self.channel.ask("http_connect", f)
if self.mode is HTTPMode.regular:
return self.handle_regular_connect(f)
elif self.mode is HTTPMode.upstream:
return self.handle_upstream_connect(f)
else:
msg = "Unexpected CONNECT request."
self.send_error_response(400, msg)
raise exceptions.ProtocolException(msg)
self.channel.ask("requestheaders", f)
if request.headers.get("expect", "").lower() == "100-continue":
# TODO: We may have to use send_response_headers for HTTP2
# here.
self.send_response(http.expect_continue_response)
request.headers.pop("expect")
request.data.content = b"".join(self.read_request_body(request))
request.timestamp_end = time.time()
validate_request_form(self.mode, request)
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e)
)
self.log("request", "debug", [repr(request)])
# set first line format to relative in regular mode,
# see https://github.com/mitmproxy/mitmproxy/issues/1759
if self.mode is HTTPMode.regular and request.first_line_format == "absolute":
request.first_line_format = "relative"
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers["Host"] = self.config.upstream_server.address.host
# Determine .scheme, .host and .port attributes for inline scripts. For
# absolute-form requests, they are directly given in the request. For
# authority-form requests, we only need to determine the request
# scheme. For relative-form requests, we need to determine host and
# port as well.
if self.mode is HTTPMode.transparent:
# Setting request.host also updates the host header, which we want
# to preserve
host_header = f.request.headers.get("host", None)
f.request.host = self.__initial_server_conn.address.host
f.request.port = self.__initial_server_conn.address.port
if host_header:
f.request.headers["host"] = host_header
f.request.scheme = "https" if self.__initial_server_tls else "http"
self.channel.ask("request", f)
try:
if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(
f.request.host,
f.request.port,
f.request.scheme
)
def get_response():
self.send_request(f.request)
f.response = self.read_response_headers()
try:
get_response()
except exceptions.NetlibException as e:
self.log(
"server communication error: %s" % repr(e),
level="debug"
)
# In any case, we try to reconnect at least once. This is
# necessary because it might be possible that we already
# initiated an upstream connection after clientconnect that
# has already been expired, e.g consider the following event
# log:
# > clientconnect (transparent mode destination known)
# > serverconnect (required for client tls handshake)
# > read n% of large request
# > server detects timeout, disconnects
# > read (100-n)% of large request
# > send large request upstream
if isinstance(e, exceptions.Http2ProtocolException):
# do not try to reconnect for HTTP2
raise exceptions.ProtocolException(
"First and only attempt to get response via HTTP2 failed."
)
self.disconnect()
self.connect()
get_response()
# call the appropriate script hook - this is an opportunity for
# an inline script to set f.stream = True
self.channel.ask("responseheaders", f)
if f.response.stream:
f.response.data.content = None
else:
f.response.data.content = b"".join(
self.read_response_body(f.request, f.response)
)
f.response.timestamp_end = time.time()
# no further manipulation of self.server_conn beyond this point
# we can safely set it as the final attribute value here.
f.server_conn = self.server_conn
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
if not f.response.stream:
# no streaming:
# we already received the full response from the server and can
# send it to the client straight away.
self.send_response(f.response)
else:
# streaming:
# First send the headers and then transfer the response incrementally
self.send_response_headers(f.response)
chunks = self.read_response_body(
f.request,
f.response
)
if callable(f.response.stream):
chunks = f.response.stream(chunks)
self.send_response_body(f.response, chunks)
f.response.timestamp_end = time.time()
if self.check_close_connection(f):
return False
# Handle 101 Switching Protocols
if f.response.status_code == 101:
# Handle a successful HTTP 101 Switching Protocols Response,
# received after e.g. a WebSocket upgrade request.
# Check for WebSockets handshake
is_websockets = (
websockets.check_handshake(f.request.headers) and
websockets.check_handshake(f.response.headers)
)
if is_websockets and not self.config.options.websockets:
self.log(
"Client requested WebSocket connection, but the protocol is disabled.",
"info"
)
if is_websockets and self.config.options.websockets:
layer = pwebsockets.WebSocketsLayer(self, f)
else:
layer = self.ctx.next_layer(self)
layer()
return False # should never be reached
except (exceptions.ProtocolException, exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return False
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e)
)
finally:
if f:
f.live = False
return True
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
first_line_format_backup = r.first_line_format
server = None
global new, cur_cycle, cur_group
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(
self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request(
(r.data.host, r.port))
server.wfile.write(
http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(server.rfile,
connect_request,
body_size_limit=bsl)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request")
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http.HTTPResponse.wrap(
http1.read_response(server.rfile, r, body_size_limit=bsl))
response_reply = self.channel.ask("response", f)
#new.append(f) #record the response
cur_cycle[cur_group] = f
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log", log.LogEntry("Connection killed", "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.first_line_format = first_line_format_backup
f.live = False
if server.connected():
server.finish()
server.close()
def _load_http_error(o: http_pb2.HTTPError) -> typing.Optional[flow.Error]:
d = {}
for m in ['msg', 'timestamp']:
if hasattr(o, m) and getattr(o, m):
d[m] = getattr(o, m)
return None if not d else flow.Error(**d)
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
f = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
try:
request = self.get_request_from_client(f)
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e))
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return
f.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
except (exceptions.ProtocolException,
exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers[
"Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
f.request.headers[
"Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(f)
try:
if websockets.check_handshake(
request.headers) and websockets.check_client_version(
request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(f.request.host,
f.request.port,
f.request.scheme)
self.get_response_from_server(f)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
self.send_response_to_client(f)
if self.check_close_connection(f):
return
# Handle 101 Switching Protocols
if f.response.status_code == 101:
return self.handle_101_switching_protocols(f)
# Upstream Proxy Mode: Handle CONNECT
if f.request.first_line_format == "authority" and f.response.status_code == 200:
self.handle_upstream_mode_connect(f.request.copy())
return
except (exceptions.ProtocolException,
exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e))
finally:
if f:
f.live = False
def test_repr(self):
e = flow.Error("yay")
assert repr(e)
assert str(e)
def terr(content="error"):
"""
@return: mitmproxy.proxy.protocol.primitives.Error
"""
err = flow.Error(content)
return err
def check_body_size(self, request: bool) -> layer.CommandGenerator[bool]:
"""
Check if the body size exceeds limits imposed by stream_large_bodies or body_size_limit.
Returns `True` if the body size exceeds body_size_limit and further processing should be stopped.
"""
if not (self.context.options.stream_large_bodies
or self.context.options.body_size_limit):
return False
# Step 1: Determine the expected body size. This can either come from a known content-length header,
# or from the amount of currently buffered bytes (e.g. for chunked encoding).
response = not request
expected_size: Optional[int]
# the 'late' case: we already started consuming the body
if request and self.request_body_buf:
expected_size = len(self.request_body_buf)
elif response and self.response_body_buf:
expected_size = len(self.response_body_buf)
else:
# the 'early' case: we have not started consuming the body
try:
expected_size = expected_http_body_size(
self.flow.request,
self.flow.response if response else None)
except ValueError: # pragma: no cover
# we just don't stream/kill malformed content-length headers.
expected_size = None
if expected_size is None or expected_size <= 0:
return False
# Step 2: Do we need to abort this?
max_total_size = human.parse_size(self.context.options.body_size_limit)
if max_total_size is not None and expected_size > max_total_size:
if request and not self.request_body_buf:
yield HttpRequestHeadersHook(self.flow)
if response and not self.response_body_buf:
yield HttpResponseHeadersHook(self.flow)
err_msg = f"{'Request' if request else 'Response'} body exceeds mitmproxy's body_size_limit."
err_code = 413 if request else 502
self.flow.error = flow.Error(err_msg)
yield HttpErrorHook(self.flow)
yield SendHttp(
ResponseProtocolError(self.stream_id, err_msg, err_code),
self.context.client)
self.client_state = self.state_errored
if response:
yield SendHttp(
RequestProtocolError(self.stream_id, err_msg, err_code),
self.context.server)
self.server_state = self.state_errored
self.flow.live = False
return True
# Step 3: Do we need to stream this?
max_stream_size = human.parse_size(
self.context.options.stream_large_bodies)
if max_stream_size is not None and expected_size > max_stream_size:
if request:
self.flow.request.stream = True
if self.request_body_buf:
# clear buffer and then fake a DataReceived event with everything we had in the buffer so far.
body_buf = self.request_body_buf
self.request_body_buf = b""
yield from self.start_request_stream()
yield from self.handle_event(
RequestData(self.stream_id, body_buf))
if response:
assert self.flow.response
self.flow.response.stream = True
if self.response_body_buf:
body_buf = self.response_body_buf
self.response_body_buf = b""
yield from self.start_response_stream()
yield from self.handle_event(
ResponseData(self.stream_id, body_buf))
return False
def terr(content: str = "error") -> flow.Error:
err = flow.Error(content, 946681207)
return err
def run(self):
r = self.f.request
first_line_format_backup = r.first_line_format
server = None
try:
self.f.response = None
# If we have a channel, run script hooks.
if self.channel:
request_reply = self.channel.ask("request", self.f)
if isinstance(request_reply, http.HTTPResponse):
self.f.response = request_reply
if not self.f.response:
# In all modes, we directly connect to the server displayed
if self.config.options.mode == "upstream":
server_address = self.config.upstream_server.address
server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0))
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request((r.data.host, r.port))
server.wfile.write(http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(
server.rfile,
connect_request,
body_size_limit=self.config.options.body_size_limit
)
if resp.status_code != 200:
raise exceptions.ReplayException("Upstream server refuses CONNECT request")
server.establish_ssl(
self.config.clientcerts,
sni=self.f.server_conn.sni
)
r.first_line_format = "relative"
else:
r.first_line_format = "absolute"
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(
server_address,
(self.config.options.listen_host, 0)
)
server.connect()
if r.scheme == "https":
server.establish_ssl(
self.config.clientcerts,
sni=self.f.server_conn.sni
)
r.first_line_format = "relative"
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
self.f.server_conn = server
self.f.response = http.HTTPResponse.wrap(
http1.read_response(
server.rfile,
r,
body_size_limit=self.config.options.body_size_limit
)
)
if self.channel:
response_reply = self.channel.ask("response", self.f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
self.f.error = flow.Error(str(e))
if self.channel:
self.channel.ask("error", self.f)
except exceptions.Kill:
# Kill should only be raised if there's a channel in the
# first place.
self.channel.tell(
"log",
log.LogEntry("Connection killed", "info")
)
except Exception:
self.channel.tell(
"log",
log.LogEntry(traceback.format_exc(), "error")
)
finally:
r.first_line_format = first_line_format_backup
self.f.live = False
if server.connected():
server.finish()
def relay_messages(
self,
event: events.ConnectionEvent) -> layer.CommandGenerator[None]:
from_client = event.connection == self.context.client
from_str = 'client' if from_client else 'server'
if from_client:
src_ws = self.client_ws
dst_ws = self.server_ws
else:
src_ws = self.server_ws
dst_ws = self.client_ws
if isinstance(event, events.DataReceived):
src_ws.receive_data(event.data)
elif isinstance(event, events.ConnectionClosed):
src_ws.receive_data(None)
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event}")
for ws_event in src_ws.events():
if isinstance(ws_event, wsproto.events.Message):
src_ws.frame_buf.append(ws_event.data)
if ws_event.message_finished:
if isinstance(ws_event, wsproto.events.TextMessage):
frame_type = Opcode.TEXT
content = "".join(src_ws.frame_buf) # type: ignore
else:
frame_type = Opcode.BINARY
content = b"".join(src_ws.frame_buf) # type: ignore
fragmentizer = Fragmentizer(src_ws.frame_buf)
src_ws.frame_buf.clear()
message = websocket.WebSocketMessage(
frame_type, from_client, content)
self.flow.messages.append(message)
yield WebsocketMessageHook(self.flow)
assert not message.killed # this is deprecated, instead we should have .content set to emptystr.
for msg in fragmentizer(message.content):
yield dst_ws.send2(msg)
elif isinstance(ws_event,
(wsproto.events.Ping, wsproto.events.Pong)):
yield commands.Log(
f"Received WebSocket {ws_event.__class__.__name__.lower()} from {from_str} "
f"(payload: {bytes(ws_event.payload)!r})")
yield dst_ws.send2(ws_event)
elif isinstance(ws_event, wsproto.events.CloseConnection):
self.flow.close_sender = from_str
self.flow.close_code = ws_event.code
self.flow.close_reason = ws_event.reason
for ws in [self.server_ws, self.client_ws]:
if ws.state in {
ConnectionState.OPEN,
ConnectionState.REMOTE_CLOSING
}:
# response == original event, so no need to differentiate here.
yield ws.send2(ws_event)
yield commands.CloseConnection(ws.conn)
if ws_event.code in {1000, 1001, 1005}:
yield WebsocketEndHook(self.flow)
else:
self.flow.error = flow.Error(
f"WebSocket Error: {format_close_event(ws_event)}")
yield WebsocketErrorHook(self.flow)
self._handle_event = self.done
else: # pragma: no cover
raise AssertionError(f"Unexpected WebSocket event: {ws_event}")
def replay(self, f): # pragma: no cover
f.live = True
r = f.request
bsl = human.parse_size(self.options.body_size_limit)
authority_backup = r.authority
server = None
try:
f.response = None
# If we have a channel, run script hooks.
request_reply = self.channel.ask("request", f)
if isinstance(request_reply, http.HTTPResponse):
f.response = request_reply
if not f.response:
# In all modes, we directly connect to the server displayed
if self.options.mode.startswith("upstream:"):
server_address = server_spec.parse_with_mode(
self.options.mode)[1].address
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
connect_request = http.make_connect_request(
(r.data.host, r.port))
server.wfile.write(
http1.assemble_request(connect_request))
server.wfile.flush()
resp = http1.read_response(server.rfile,
connect_request,
body_size_limit=bsl)
if resp.status_code != 200:
raise exceptions.ReplayException(
"Upstream server refuses CONNECT request")
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.authority = b""
else:
r.authority = hostport(r.scheme, r.host, r.port)
else:
server_address = (r.host, r.port)
server = connections.ServerConnection(server_address)
server.connect()
if r.scheme == "https":
server.establish_tls(
sni=f.server_conn.sni,
**tls.client_arguments_from_options(self.options))
r.authority = ""
server.wfile.write(http1.assemble_request(r))
server.wfile.flush()
r.timestamp_start = r.timestamp_end = time.time()
if f.server_conn:
f.server_conn.close()
f.server_conn = server
f.response = http1.read_response(server.rfile,
r,
body_size_limit=bsl)
response_reply = self.channel.ask("response", f)
if response_reply == exceptions.Kill:
raise exceptions.Kill()
except (exceptions.ReplayException, exceptions.NetlibException) as e:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
except exceptions.Kill:
self.channel.tell("log",
log.LogEntry(flow.Error.KILLED_MESSAGE, "info"))
except Exception as e:
self.channel.tell("log", log.LogEntry(repr(e), "error"))
finally:
r.authority = authority_backup
f.live = False
if server and server.connected():
server.finish()
server.close()
def kill(self, force=False): # pragma: no cover
warnings.warn(
"reply.kill() is deprecated, use flow.kill() or set the error attribute instead.",
DeprecationWarning,
stacklevel=2)
self.obj.error = flow.Error(flow.Error.KILLED_MESSAGE)
