def twebsocketflow(client_conn=True,
server_conn=True,
messages=True,
err=None,
handshake_flow=True):
if client_conn is True:
client_conn = tclient_conn()
if server_conn is True:
server_conn = tserver_conn()
if handshake_flow is True:
req = http.HTTPRequest("relative",
"GET",
"http",
"example.com",
"80",
"/ws",
"HTTP/1.1",
headers=net_http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'')
resp = http.HTTPResponse(
"HTTP/1.1",
101,
reason=net_http.status_codes.RESPONSES.get(101),
headers=net_http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
),
content=b'',
)
handshake_flow = http.HTTPFlow(client_conn, server_conn)
handshake_flow.request = req
handshake_flow.response = resp
f = websocket.WebSocketFlow(client_conn, server_conn, handshake_flow)
handshake_flow.metadata['websocket_flow'] = f
if messages is True:
messages = [
websocket.WebSocketMessage(websockets.OPCODE.BINARY, True,
b"hello binary"),
websocket.WebSocketMessage(websockets.OPCODE.TEXT, True,
"hello text".encode()),
websocket.WebSocketMessage(websockets.OPCODE.TEXT, False,
"it's me".encode()),
]
if err is True:
err = terr()
f.messages = messages
f.error = err
f.reply = controller.DummyReply()
return f
def test_view_auto():
v = full_eval(auto.ViewAuto())
f = v(b"foo", headers=http.Headers())
assert f[0] == "Raw"
f = v(b"<html></html>", headers=http.Headers(content_type="text/html"))
assert f[0] == "HTML"
f = v(b"foo", headers=http.Headers(content_type="text/flibble"))
assert f[0] == "Raw"
f = v(b"<xml></xml>", headers=http.Headers(content_type="text/flibble"))
assert f[0].startswith("XML")
f = v(b"<svg></svg>", headers=http.Headers(content_type="image/svg+xml"))
assert f[0].startswith("XML")
f = v(b"verybinary",
headers=http.Headers(content_type="image/new-magic-image-format"))
assert f[0] == "Unknown Image"
f = v(b"\xFF" * 30)
assert f[0] == "Hex"
f = v(b"", headers=http.Headers())
assert f[0] == "No content"
f = v(
b"",
headers=http.Headers(),
query=multidict.MultiDict([("foo", "bar")]),
)
assert f[0] == "Query"
def make_error_response(status_code, message, headers=None):
response = http.status_codes.RESPONSES.get(status_code, "Unknown")
body = """
<html>
<head>
<title>%d %s</title>
</head>
<body>%s</body>
</html>
""".strip() % (status_code, response, cgi.escape(message))
body = body.encode("utf8", "replace")
if not headers:
headers = http.Headers(
Server=version.MITMPROXY,
Connection="close",
Content_Length=str(len(body)),
Content_Type="text/html"
)
return HTTPResponse(
b"HTTP/1.1",
status_code,
response,
headers,
body,
)
def client_handshake_headers(version=None,
key=None,
protocol=None,
extensions=None):
"""
Create the headers for a valid HTTP upgrade request. If Key is not
specified, it is generated, and can be found in sec-websocket-key in
the returned header set.
Returns an instance of http.Headers
"""
if version is None:
version = VERSION
if key is None:
key = base64.b64encode(os.urandom(16)).decode('ascii')
h = http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version=version,
sec_websocket_key=key,
)
if protocol is not None:
h['sec-websocket-protocol'] = protocol
if extensions is not None:
h['sec-websocket-extensions'] = extensions
return h
def handle(self):
try:
request = http.http1.read_request(self.rfile)
assert websocket_utils.check_handshake(request.headers)
response = http.Response(
http_version=b"HTTP/1.1",
status_code=101,
reason=http.status_codes.RESPONSES.get(101).encode(),
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
sec_websocket_extensions='permessage-deflate' if "permessage-deflate" in request.headers.values() else ''
),
content=b'',
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
self.wfile.write(http.http1.assemble_response(response))
self.wfile.flush()
self.server.handle_websockets(self.rfile, self.wfile)
except:
traceback.print_exc()
def make_error_response(
status_code: int,
message: str = "",
headers: Optional[http.Headers] = None,
) -> HTTPResponse:
body: bytes = """
<html>
<head>
<title>{status_code} {reason}</title>
</head>
<body>
<h1>{status_code} {reason}</h1>
<p>{message}</p>
</body>
</html>
""".strip().format(
status_code=status_code,
reason=http.status_codes.RESPONSES.get(status_code, "Unknown"),
message=html.escape(message),
).encode("utf8", "replace")
if not headers:
headers = http.Headers(Server=version.MITMPROXY,
Connection="close",
Content_Length=str(len(body)),
Content_Type="text/html")
return HTTPResponse.make(status_code, body, headers)
def test_replay(self):
opts = options.Options()
fm = master.Master(opts)
f = tflow.tflow(resp=True)
f.request.content = None
with pytest.raises(ReplayException, match="missing"):
fm.replay_request(f)
f.request = None
with pytest.raises(ReplayException, match="request"):
fm.replay_request(f)
f.intercepted = True
with pytest.raises(ReplayException, match="intercepted"):
fm.replay_request(f)
f.live = True
with pytest.raises(ReplayException, match="live"):
fm.replay_request(f)
req = tutils.treq(headers=net_http.Headers(((b":authority", b"foo"), (b"header", b"qvalue"), (b"content-length", b"7"))))
f = tflow.tflow(req=req)
f.request.http_version = "HTTP/2.0"
with mock.patch('mitmproxy.proxy.protocol.http_replay.RequestReplayThread.run'):
rt = fm.replay_request(f)
assert rt.f.request.http_version == "HTTP/1.1"
assert ":authority" not in rt.f.request.headers
def test_client_handshake_headers(_):
assert websocket.client_handshake_headers() == \
http.Headers([
(b'connection', b'upgrade'),
(b'upgrade', b'websocket'),
(b'sec-websocket-version', b'13'),
(b'sec-websocket-key', b'cHVtcGtpbnNwdW1wa2lucw=='),
])
assert websocket.client_handshake_headers(b"13", b"foobar", b"foo", b"bar") == \
http.Headers([
(b'connection', b'upgrade'),
(b'upgrade', b'websocket'),
(b'sec-websocket-version', b'13'),
(b'sec-websocket-key', b'foobar'),
(b'sec-websocket-protocol', b'foo'),
(b'sec-websocket-extensions', b'bar')
])
def test_server_handshake_headers():
assert websocket.server_handshake_headers("foobar", "foo", "bar") == \
http.Headers([
(b'connection', b'upgrade'),
(b'upgrade', b'websocket'),
(b'sec-websocket-accept', b'AzhRPA4TNwR6I/riJheN0TfR7+I='),
(b'sec-websocket-protocol', b'foo'),
(b'sec-websocket-extensions', b'bar'),
])
def test_with_body(self):
bytes = HTTP2StateProtocol(self.c, is_server=True).assemble_response(
http.Response(b"HTTP/2.0", 200, b'', http.Headers(foo=b"bar"),
b'foobar'))
assert len(bytes) == 2
assert bytes[0] ==\
codecs.decode('00000901040000000288408294e7838c767f', 'hex_codec')
assert bytes[1] ==\
codecs.decode('000006000100000002666f6f626172', 'hex_codec')
def make_connect_response(http_version):
# Do not send any response headers as it breaks proxying non-80 ports on
# Android emulators using the -http-proxy option.
return HTTPResponse(
http_version,
200,
b"Connection established",
http.Headers(),
b"",
)
def test_view_multipart():
view = full_eval(multipart.ViewMultipart())
v = b"""
--AaB03x
Content-Disposition: form-data; name="submit-name"
Larry
--AaB03x
""".strip()
h = http.Headers(content_type="multipart/form-data; boundary=AaB03x")
assert view(v, headers=h)
h = http.Headers()
assert not view(v, headers=h)
h = http.Headers(content_type="multipart/form-data")
assert not view(v, headers=h)
h = http.Headers(content_type="unparseable")
assert not view(v, headers=h)
def start_response(status, headers, exc_info=None):
if exc_info:
if state["headers_sent"]:
raise exc_info[1]
elif state["status"]:
raise AssertionError('Response already started')
state["status"] = status
state["headers"] = http.Headers([[strutils.always_bytes(k), strutils.always_bytes(v)] for k, v in headers])
if exc_info:
self.error_page(soc, state["headers_sent"], traceback.format_tb(exc_info[2]))
state["headers_sent"] = True
def test_create_headers_multiple_frames(self):
headers = http.Headers([(b':method', b'GET'), (b':path', b'/'),
(b':scheme', b'https'), (b'foo', b'bar'),
(b'server', b'version')])
protocol = HTTP2StateProtocol(self.c)
protocol.http2_settings[
hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 8
data = protocol._create_headers(headers, 1, end_stream=True)
assert len(data) == 3
assert data[0] == bytes.fromhex("000008010100000001828487408294e783")
assert data[1] == bytes.fromhex("0000080900000000018c767f7685ee5b10")
assert data[2] == bytes.fromhex("00000209040000000163d5")
def test_http2(self):
cp = clientplayback.ClientPlayback()
with taddons.context(cp):
req = tutils.treq(headers=net_http.Headers(((b":authority",
b"foo"), (b"header",
b"qvalue"),
(b"content-length",
b"7"))))
f = tflow.tflow(req=req)
f.request.http_version = "HTTP/2.0"
cp.start_replay([f])
assert f.request.http_version == "HTTP/1.1"
assert ":authority" not in f.request.headers
def test_create_headers(self):
headers = http.Headers([
(b':method', b'GET'),
(b':path', b'index.html'),
(b':scheme', b'https'),
(b'foo', b'bar')])
data = HTTP2StateProtocol(self.c)._create_headers(
headers, 1, end_stream=True)
assert b''.join(data) == bytes.fromhex("000014010500000001824488355217caf3a69a3f87408294e7838c767f")
data = HTTP2StateProtocol(self.c)._create_headers(
headers, 1, end_stream=False)
assert b''.join(data) == bytes.fromhex("000014010400000001824488355217caf3a69a3f87408294e7838c767f")
def test_with_body(self):
data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(http.Response(
http_version=b"HTTP/2.0",
status_code=200,
reason=b'',
headers=http.Headers(foo=b"bar"),
content=b'foobar',
trailers=None,
timestamp_start=0,
timestamp_end=0,
))
assert len(data) == 2
assert data[0] == bytes.fromhex("00000901040000000288408294e7838c767f")
assert data[1] == bytes.fromhex("000006000100000002666f6f626172")
def test_create_headers_multiple_frames(self):
headers = http.Headers([
(b':method', b'GET'),
(b':path', b'/'),
(b':scheme', b'https'),
(b'foo', b'bar'),
(b'server', b'version')])
protocol = HTTP2StateProtocol(self.c)
protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 8
bytes = protocol._create_headers(headers, 1, end_stream=True)
assert len(bytes) == 3
assert bytes[0] == codecs.decode('000008010100000001828487408294e783', 'hex_codec')
assert bytes[1] == codecs.decode('0000080900000000018c767f7685ee5b10', 'hex_codec')
assert bytes[2] == codecs.decode('00000209040000000163d5', 'hex_codec')
def make_connect_request(address: Tuple[str, int]) -> HTTPRequest:
return HTTPRequest(
host=address[0],
port=address[1],
method=b"CONNECT",
scheme=b"",
authority=f"{address[0]}:{address[1]}".encode(),
path=b"",
http_version=b"HTTP/1.1",
headers=http.Headers(),
content=b"",
trailers=None,
timestamp_start=time.time(),
timestamp_end=time.time(),
)
def test_create_headers(self):
headers = http.Headers([
(b':method', b'GET'),
(b':path', b'index.html'),
(b':scheme', b'https'),
(b'foo', b'bar')])
bytes = HTTP2StateProtocol(self.c)._create_headers(
headers, 1, end_stream=True)
assert b''.join(bytes) ==\
codecs.decode('000014010500000001824488355217caf3a69a3f87408294e7838c767f', 'hex_codec')
bytes = HTTP2StateProtocol(self.c)._create_headers(
headers, 1, end_stream=False)
assert b''.join(bytes) ==\
codecs.decode('000014010400000001824488355217caf3a69a3f87408294e7838c767f', 'hex_codec')
def server_handshake_headers(client_key, protocol=None, extensions=None):
"""
The server response is a valid HTTP 101 response.
Returns an instance of http.Headers
"""
h = http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_accept=create_server_nonce(client_key),
)
if protocol is not None:
h['sec-websocket-protocol'] = protocol
if extensions is not None:
h['sec-websocket-extensions'] = extensions
return h
def tresp(**kwargs):
"""
Returns:
mitmproxy.net.http.Response
"""
default = dict(
http_version=b"HTTP/1.1",
status_code=200,
reason=b"OK",
headers=http.Headers(((b"header-response", b"svalue"), (b"content-length", b"7"))),
content=b"message",
timestamp_start=time.time(),
timestamp_end=time.time(),
)
default.update(kwargs)
return http.Response(**default)
def tresp(**kwargs) -> http.Response:
"""
Returns:
mitmproxy.net.http.Response
"""
default = dict(
http_version=b"HTTP/1.1",
status_code=200,
reason=b"OK",
headers=http.Headers(((b"header-response", b"svalue"), (b"content-length", b"7"))),
content=b"message",
trailers=None,
timestamp_start=946681202,
timestamp_end=946681203,
)
default.update(kwargs)
return http.Response(**default) # type: ignore
def treq(**kwargs):
"""
Returns:
mitmproxy.net.http.Request
"""
default = dict(first_line_format="relative",
method=b"GET",
scheme=b"http",
host=b"address",
port=22,
path=b"/path",
http_version=b"HTTP/1.1",
headers=http.Headers(
((b"header", b"qvalue"), (b"content-length", b"7"))),
content=b"content")
default.update(kwargs)
return http.Request(**default)
def test_request_with_body(self):
bytes = HTTP2StateProtocol(self.c).assemble_request(http.Request(
b'',
b'GET',
b'https',
b'',
b'',
b'/',
b"HTTP/2.0",
http.Headers([(b'foo', b'bar')]),
b'foobar',
))
assert len(bytes) == 2
assert bytes[0] ==\
codecs.decode('0000150104000000018284874188089d5c0b8170dc07408294e7838c767f', 'hex_codec')
assert bytes[1] ==\
codecs.decode('000006000100000001666f6f626172', 'hex_codec')
def test_request_with_body(self):
data = HTTP2StateProtocol(self.c).assemble_request(http.Request(
host="",
port=0,
method=b'GET',
scheme=b'https',
authority=b'',
path=b'/',
http_version=b"HTTP/2.0",
headers=http.Headers([(b'foo', b'bar')]),
content=b'foobar',
trailers=None,
timestamp_start=0,
timestamp_end=None,
))
assert len(data) == 2
assert data[0] == bytes.fromhex("0000150104000000018284874188089d5c0b8170dc07408294e7838c767f")
assert data[1] == bytes.fromhex("000006000100000001666f6f626172")
def split_pseudo_headers(
h2_headers: Sequence[Tuple[bytes, bytes]]
) -> Tuple[Dict[bytes, bytes], net_http.Headers]:
pseudo_headers: Dict[bytes, bytes] = {}
i = 0
for (header, value) in h2_headers:
if header.startswith(b":"):
if header in pseudo_headers:
raise ValueError(f"Duplicate HTTP/2 pseudo header: {header!r}")
pseudo_headers[header] = value
i += 1
else:
# Pseudo-headers must be at the start, we are done here.
break
headers = net_http.Headers(h2_headers[i:])
return pseudo_headers, headers
def _setup_connection(self):
client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
request = http.Request(
"authority",
"CONNECT",
"",
"127.0.0.1",
self.server.server.address[1],
"",
"HTTP/1.1",
content=b'')
client.wfile.write(http.http1.assemble_request(request))
client.wfile.flush()
response = http.http1.read_response(client.rfile, request)
if self.ssl:
client.convert_to_ssl()
assert client.ssl_established
request = http.Request(
"relative",
"GET",
"http",
"127.0.0.1",
self.server.server.address[1],
"/ws",
"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
),
content=b'')
client.wfile.write(http.http1.assemble_request(request))
client.wfile.flush()
response = http.http1.read_response(client.rfile, request)
assert websockets.check_handshake(response.headers)
return client
def setup_connection(self, extension=False):
self.client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
self.client.connect()
request = http.Request(
"authority",
"CONNECT",
"",
"127.0.0.1",
self.server.server.address[1],
"",
"HTTP/1.1",
content=b'')
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
if self.ssl:
self.client.convert_to_tls()
assert self.client.tls_established
request = http.Request(
"relative",
"GET",
"http",
"127.0.0.1",
self.server.server.address[1],
"/ws",
"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
sec_websocket_extensions="permessage-deflate" if extension else ""
),
content=b'')
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
assert websockets.check_handshake(response.headers)
def treq(**kwargs) -> http.Request:
"""
Returns:
mitmproxy.net.http.Request
"""
default = dict(
host="address",
port=22,
method=b"GET",
scheme=b"http",
authority=b"",
path=b"/path",
http_version=b"HTTP/1.1",
headers=http.Headers(((b"header", b"qvalue"), (b"content-length", b"7"))),
content=b"content",
trailers=None,
timestamp_start=946681200,
timestamp_end=946681201,
)
default.update(kwargs)
return http.Request(**default) # type: ignore
