def tflow(
*,
client_conn: Optional[connection.Client] = None,
server_conn: Optional[connection.Server] = None,
req: Optional[http.Request] = None,
resp: Union[bool, http.Response] = False,
err: Union[bool, flow.Error] = False,
ws: Union[bool, websocket.WebSocketData] = False,
) -> http.HTTPFlow:
"""Create a flow for testing."""
if client_conn is None:
client_conn = tclient_conn()
if server_conn is None:
server_conn = tserver_conn()
if req is None:
req = treq()
if resp is True:
resp = tresp()
if err is True:
err = terr()
if ws is True:
ws = twebsocket()
assert resp is False or isinstance(resp, http.Response)
assert err is False or isinstance(err, flow.Error)
assert ws is False or isinstance(ws, websocket.WebSocketData)
f = http.HTTPFlow(client_conn, server_conn)
f.request = req
f.response = resp or None
f.error = err or None
f.websocket = ws or None
f.reply = controller.DummyReply()
return f
def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None):
"""
@type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection
@type server_conn: bool | None | mitmproxy.proxy.connection.ServerConnection
@type req: bool | None | mitmproxy.proxy.protocol.http.HTTPRequest
@type resp: bool | None | mitmproxy.proxy.protocol.http.HTTPResponse
@type err: bool | None | mitmproxy.proxy.protocol.primitives.Error
@return: mitmproxy.proxy.protocol.http.HTTPFlow
"""
if client_conn is True:
client_conn = tclient_conn()
if server_conn is True:
server_conn = tserver_conn()
if req is True:
req = tutils.treq()
if resp is True:
resp = tutils.tresp()
if err is True:
err = terr()
if req:
req = http.HTTPRequest.wrap(req)
if resp:
resp = http.HTTPResponse.wrap(resp)
f = http.HTTPFlow(client_conn, server_conn)
f.request = req
f.response = resp
f.error = err
f.reply = controller.DummyReply()
return f
async def load_flow(self, f):
"""
Loads a flow and links websocket & handshake flows
"""
if isinstance(f, http.HTTPFlow):
self._change_reverse_host(f)
if 'websocket' in f.metadata:
self.waiting_flows.append(f)
if isinstance(f, websocket.WebSocketFlow):
hfs = [
hf for hf in self.waiting_flows
if hf.id == f.metadata['websocket_handshake']
]
if hfs:
hf = hfs[0]
f.handshake_flow = hf
self.waiting_flows.remove(hf)
self._change_reverse_host(f.handshake_flow)
else:
# this will fail - but at least it will load the remaining flows
f.handshake_flow = http.HTTPFlow(None, None)
f.reply = controller.DummyReply()
for e, o in eventsequence.iterate(f):
await self.addons.handle_lifecycle(e, o)
def twebsocketflow(client_conn=True,
server_conn=True,
messages=True,
err=None,
handshake_flow=True):
if client_conn is True:
client_conn = tclient_conn()
if server_conn is True:
server_conn = tserver_conn()
if handshake_flow is True:
req = http.HTTPRequest("relative",
"GET",
"http",
"example.com",
"80",
"/ws",
"HTTP/1.1",
headers=net_http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'')
resp = http.HTTPResponse(
"HTTP/1.1",
101,
reason=net_http.status_codes.RESPONSES.get(101),
headers=net_http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
)
handshake_flow = http.HTTPFlow(client_conn, server_conn)
handshake_flow.request = req
handshake_flow.response = resp
f = websocket.WebSocketFlow(client_conn, server_conn, handshake_flow)
handshake_flow.metadata['websocket_flow'] = f
if messages is True:
messages = [
websocket.WebSocketMessage(websockets.OPCODE.BINARY, True,
b"hello binary"),
websocket.WebSocketMessage(websockets.OPCODE.TEXT, True,
"hello text".encode()),
websocket.WebSocketMessage(websockets.OPCODE.TEXT, False,
"it's me".encode()),
]
if err is True:
err = terr()
f.messages = messages
f.error = err
f.reply = controller.DummyReply()
return f
def create(self, method: str, url: str) -> None:
req = http.HTTPRequest.make(method.upper(), url)
c = connections.ClientConnection.make_dummy(("", 0))
s = connections.ServerConnection.make_dummy((req.host, req.port))
f = http.HTTPFlow(c, s)
f.request = req
f.request.headers["Host"] = req.host
self.add([f])
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
self.flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
if not self._process_flow(self.flow):
return
def twebsocketflow(messages=True,
err=None,
close_code=None,
close_reason='') -> http.HTTPFlow:
flow = http.HTTPFlow(tclient_conn(), tserver_conn())
flow.request = http.Request(
"example.com",
80,
b"GET",
b"http",
b"example.com",
b"/ws",
b"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'',
trailers=None,
timestamp_start=946681200,
timestamp_end=946681201,
)
flow.response = http.Response(
b"HTTP/1.1",
101,
reason=b"Switching Protocols",
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
trailers=None,
timestamp_start=946681202,
timestamp_end=946681203,
)
flow.websocket = twebsocket()
flow.websocket.close_reason = close_reason
if close_code is not None:
flow.websocket.close_code = close_code
else:
if err is True:
# ABNORMAL_CLOSURE
flow.websocket.close_code = 1006
else:
# NORMAL_CLOSURE
flow.websocket.close_code = 1000
flow.reply = controller.DummyReply()
return flow
def create(self, method: str, url: str) -> None:
try:
req = http.HTTPRequest.make(method.upper(), url)
except ValueError as e:
raise exceptions.CommandError("Invalid URL: %s" % e)
c = connections.ClientConnection.make_dummy(("", 0))
s = connections.ServerConnection.make_dummy((req.host, req.port))
f = http.HTTPFlow(c, s)
f.request = req
f.request.headers["Host"] = req.host
self.add([f])
def __call__(self):
if self.mode == HTTPMode.transparent:
self.__initial_server_tls = self.server_tls
self.__initial_server_address = self.server_conn.address
while True:
flow = http.HTTPFlow(self.client_conn,
self.server_conn,
live=self,
mode=self.mode.name)
if not self._process_flow(flow):
return
def create(self, method: str, url: str) -> None:
try:
req = http.HTTPRequest.make(method.upper(), url)
except ValueError as e:
raise exceptions.CommandError("Invalid URL: %s" % e)
c = context.Client(("", 0), ("", 0), req.timestamp_start - 0.0001)
s = context.Server((req.host, req.port))
f = http.HTTPFlow(c, s)
f.request = req
f.request.headers["Host"] = req.host
self.add([f])
def state_wait_for_request_headers(
self, event: RequestHeaders) -> layer.CommandGenerator[None]:
if not event.replay_flow:
self.flow = http.HTTPFlow(self.context.client, self.context.server)
else:
self.flow = event.replay_flow
self.flow.request = event.request
if err := validate_request(self.mode, self.flow.request):
self.flow.response = http.HTTPResponse.make(502, str(err))
self.client_state = self.state_errored
return (yield from self.send_response())
def twebsocketflow(messages=True, err=None) -> http.HTTPFlow:
flow = http.HTTPFlow(tclient_conn(), tserver_conn())
flow.request = http.Request(
"example.com",
80,
b"GET",
b"http",
b"example.com",
b"/ws",
b"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'',
trailers=None,
timestamp_start=946681200,
timestamp_end=946681201,
)
flow.response = http.Response(
b"HTTP/1.1",
101,
reason=b"Switching Protocols",
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
trailers=None,
timestamp_start=946681202,
timestamp_end=946681203,
)
flow.websocket = websocket.WebSocketData()
if messages is True:
flow.websocket.messages = [
websocket.WebSocketMessage(Opcode.BINARY, True, b"hello binary",
946681203),
websocket.WebSocketMessage(Opcode.TEXT, True, b"hello text",
946681204),
websocket.WebSocketMessage(Opcode.TEXT, False, b"it's me",
946681205),
]
if err is True:
flow.error = terr()
flow.reply = controller.DummyReply()
return flow
def create_request(self, method, scheme, host, port, path):
"""
this method creates a new artificial and minimalist request also adds it to flowlist
"""
c = connections.ClientConnection.make_dummy(("", 0))
s = connections.ServerConnection.make_dummy((host, port))
f = http.HTTPFlow(c, s)
headers = mitmproxy.net.http.Headers()
req = http.HTTPRequest("absolute", method, scheme, host, port, path,
b"HTTP/1.1", headers, b"")
f.request = req
self.load_flow(f)
return f
def create_request(self, method, url):
"""
Create a new artificial and minimalist request also adds it to flowlist.
Raises:
ValueError, if the url is malformed.
"""
req = http.HTTPRequest.make(method, url)
c = connections.ClientConnection.make_dummy(("", 0))
s = connections.ServerConnection.make_dummy((req.host, req.port))
f = http.HTTPFlow(c, s)
f.request = req
self.load_flow(f)
return f
def create(self, method: str, url: str) -> None:
try:
req = http.HTTPRequest.make(method.upper(), url)
except ValueError as e:
raise exceptions.CommandError("Invalid URL: %s" % e)
if compat.new_proxy_core: # pragma: no cover
c = compat.Client(("", 0), ("", 0), req.timestamp_start - 0.0001)
s = compat.Server((req.host, req.port))
else: # pragma: no cover
c = connections.ClientConnection.make_dummy(("", 0))
s = connections.ServerConnection.make_dummy((req.host, req.port))
f = http.HTTPFlow(c, s)
f.request = req
f.request.headers["Host"] = req.host
self.add([f])
def __call__(self):
logging.info('http.py: server address %s', self.server_conn.address)
if self.mode == HTTPMode.transparent:
self.__initial_server_tls = self.server_tls
self.__initial_server_address = self.server_conn.address
logging.info('http.py: set self.__inital_server_address')
else:
logging.info('http.py: not setting address for mode %s', self.mode)
logging.info('HttpLayer.__call__ entering "while True" loop')
while True:
logging.info('HttpLayer.__call__ in "while True" loop')
flow = http.HTTPFlow(self.client_conn,
self.server_conn,
live=self,
mode=self.mode.name)
if not self._process_flow(flow):
logging.info('HttpLayer.__call__ done')
return
else:
logging.info('HttpLayer.__call__ succeeded')
def __call__(self):
if self.mode == HTTPMode.transparent:
self.__initial_server_tls = self.server_tls
self.__initial_server_address = self.server_conn.address
while True:
flow = http.HTTPFlow(self.client_conn,
self.server_conn,
live=self,
mode=self.mode.name)
root_ctx = self.get_root_ctx()
if hasattr(root_ctx, "flow"):
flow.metadata[
'WhiteListFiltered'] = root_ctx.flow.metadata.pop(
'WhiteListFiltered', None)
flow.metadata[
'BlackListFiltered'] = root_ctx.flow.metadata.pop(
'BlackListFiltered', None)
setattr(self.get_root_ctx(), "flow", flow)
if not self._process_flow(flow):
return
def start_handshake(self) -> layer.CommandGenerator[None]:
if not self.send_connect:
return (yield from super().start_handshake())
assert self.conn.address
flow = http.HTTPFlow(self.context.client, self.tunnel_connection)
flow.request = http.Request(
host=self.conn.address[0],
port=self.conn.address[1],
method=b"CONNECT",
scheme=b"",
authority=f"{self.conn.address[0]}:{self.conn.address[1]}".encode(
),
path=b"",
http_version=b"HTTP/1.1",
headers=http.Headers(),
content=b"",
trailers=None,
timestamp_start=time.time(),
timestamp_end=time.time(),
)
yield HttpConnectUpstreamHook(flow)
raw = http1.assemble_request(flow.request)
yield commands.SendData(self.tunnel_connection, raw)
def __call__(self):
if self.mode == "transparent":
self.__initial_server_tls = self.server_tls
self.__initial_server_conn = self.server_conn
while True:
f = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
try:
request = self.get_request_from_client(f)
# Make sure that the incoming request matches our expectations
self.validate_request(request)
except exceptions.HttpReadDisconnect:
# don't throw an error for disconnects that happen before/between requests.
return
except exceptions.HttpException as e:
# We optimistically guess there might be an HTTP client on the
# other end
self.send_error_response(400, repr(e))
raise exceptions.ProtocolException(
"HTTP protocol error in client request: {}".format(e))
self.log("request", "debug", [repr(request)])
# Handle Proxy Authentication
# Proxy Authentication conceptually does not work in transparent mode.
# We catch this misconfiguration on startup. Here, we sort out requests
# after a successful CONNECT request (which do not need to be validated anymore)
if not (self.http_authenticated or self.authenticate(request)):
return
f.request = request
try:
# Regular Proxy Mode: Handle CONNECT
if self.mode == "regular" and request.first_line_format == "authority":
self.handle_regular_mode_connect(request)
return
except (exceptions.ProtocolException,
exceptions.NetlibException) as e:
# HTTPS tasting means that ordinary errors like resolution and
# connection errors can happen here.
self.send_error_response(502, repr(e))
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return
# update host header in reverse proxy mode
if self.config.options.mode == "reverse":
f.request.headers[
"Host"] = self.config.upstream_server.address.host
# set upstream auth
if self.mode == "upstream" and self.config.upstream_auth is not None:
f.request.headers[
"Proxy-Authorization"] = self.config.upstream_auth
self.process_request_hook(f)
try:
if websockets.check_handshake(
request.headers) and websockets.check_client_version(
request.headers):
# We only support RFC6455 with WebSockets version 13
# allow inline scripts to manipulate the client handshake
self.channel.ask("websocket_handshake", f)
if not f.response:
self.establish_server_connection(f.request.host,
f.request.port,
f.request.scheme)
self.get_response_from_server(f)
else:
# response was set by an inline script.
# we now need to emulate the responseheaders hook.
self.channel.ask("responseheaders", f)
self.log("response", "debug", [repr(f.response)])
self.channel.ask("response", f)
self.send_response_to_client(f)
if self.check_close_connection(f):
return
# Handle 101 Switching Protocols
if f.response.status_code == 101:
return self.handle_101_switching_protocols(f)
# Upstream Proxy Mode: Handle CONNECT
if f.request.first_line_format == "authority" and f.response.status_code == 200:
self.handle_upstream_mode_connect(f.request.copy())
return
except (exceptions.ProtocolException,
exceptions.NetlibException) as e:
self.send_error_response(502, repr(e))
if not f.response:
f.error = flow.Error(str(e))
self.channel.ask("error", f)
return
else:
raise exceptions.ProtocolException(
"Error in HTTP connection: %s" % repr(e))
finally:
if f:
f.live = False
def twebsocketflow(client_conn=True,
server_conn=True,
messages=True,
err=None,
handshake_flow=True):
if client_conn is True:
client_conn = tclient_conn()
if server_conn is True:
server_conn = tserver_conn()
if handshake_flow is True:
req = http.HTTPRequest(
"example.com",
80,
b"GET",
b"http",
b"example.com",
b"/ws",
b"HTTP/1.1",
headers=net_http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'',
trailers=None,
timestamp_start=946681200,
timestamp_end=946681201,
)
resp = http.HTTPResponse(
b"HTTP/1.1",
101,
reason=net_http.status_codes.RESPONSES.get(101),
headers=net_http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
trailers=None,
timestamp_start=946681202,
timestamp_end=946681203,
)
handshake_flow = http.HTTPFlow(client_conn, server_conn)
handshake_flow.request = req
handshake_flow.response = resp
f = websocket.WebSocketFlow(client_conn, server_conn, handshake_flow)
f.metadata['websocket_handshake'] = handshake_flow.id
handshake_flow.metadata['websocket_flow'] = f.id
handshake_flow.metadata['websocket'] = True
if messages is True:
messages = [
websocket.WebSocketMessage(Opcode.BINARY, True, b"hello binary"),
websocket.WebSocketMessage(Opcode.TEXT, True, b"hello text"),
websocket.WebSocketMessage(Opcode.TEXT, False, b"it's me"),
]
if err is True:
err = terr()
f.messages = messages
f.error = err
f.reply = controller.DummyReply()
return f
