def test_assemble_request_line():
assert _assemble_request_line(treq().data) == b"GET /path HTTP/1.1"
authority_request = treq(method=b"CONNECT", first_line_format="authority").data
assert _assemble_request_line(authority_request) == b"CONNECT address:22 HTTP/1.1"
absolute_request = treq(first_line_format="absolute").data
assert _assemble_request_line(absolute_request) == b"GET http://address:22/path HTTP/1.1"
with raises(RuntimeError):
_assemble_request_line(treq(first_line_format="invalid_form").data)
def test_intercept(self):
"""regression test for https://github.com/mitmproxy/mitmproxy/issues/1605"""
m = self.mkmaster(intercept="~b bar")
f = tflow.tflow(req=tutils.treq(content=b"foo"))
m.addons.handle_lifecycle("request", f)
assert not m.view[0].intercepted
f = tflow.tflow(req=tutils.treq(content=b"bar"))
m.addons.handle_lifecycle("request", f)
assert m.view[1].intercepted
f = tflow.tflow(resp=tutils.tresp(content=b"bar"))
m.addons.handle_lifecycle("request", f)
assert m.view[2].intercepted
def test_assemble_request():
assert assemble_request(treq()) == (
b"GET /path HTTP/1.1\r\n"
b"header: qvalue\r\n"
b"content-length: 7\r\n"
b"host: address:22\r\n"
b"\r\n"
b"content"
)
with raises(exceptions.HttpException):
assemble_request(treq(content=None))
def test_path(self):
req = treq()
_test_decoded_attr(req, "path")
# path can also be None.
req.path = None
assert req.path is None
assert req.data.path is None
def cycle(self, master, content):
f = tflow.tflow(req=tutils.treq(content=content))
master.addons.handle_lifecycle("clientconnect", f.client_conn)
for i in eventsequence.iterate(f):
master.addons.handle_lifecycle(*i)
master.addons.handle_lifecycle("clientdisconnect", f.client_conn)
return f
def test_anticache(self):
request = treq()
request.headers["If-Modified-Since"] = "foo"
request.headers["If-None-Match"] = "bar"
request.anticache()
assert "If-Modified-Since" not in request.headers
assert "If-None-Match" not in request.headers
def test_replay(self):
opts = options.Options()
fm = master.Master(opts)
f = tflow.tflow(resp=True)
f.request.content = None
with pytest.raises(ReplayException, match="missing"):
fm.replay_request(f)
f.request = None
with pytest.raises(ReplayException, match="request"):
fm.replay_request(f)
f.intercepted = True
with pytest.raises(ReplayException, match="intercepted"):
fm.replay_request(f)
f.live = True
with pytest.raises(ReplayException, match="live"):
fm.replay_request(f)
req = tutils.treq(headers=net_http.Headers(((b":authority", b"foo"), (b"header", b"qvalue"), (b"content-length", b"7"))))
f = tflow.tflow(req=req)
f.request.http_version = "HTTP/2.0"
with mock.patch('mitmproxy.proxy.protocol.http_replay.RequestReplayThread.run'):
rt = fm.replay_request(f)
assert rt.f.request.http_version == "HTTP/1.1"
assert ":authority" not in rt.f.request.headers
def test_read_chunked():
req = treq(content=None)
req.headers["Transfer-Encoding"] = "chunked"
data = b"1\r\na\r\n0\r\n"
with raises(exceptions.HttpSyntaxException):
b"".join(_read_chunked(BytesIO(data)))
data = b"1\r\na\r\n0\r\n\r\n"
assert b"".join(_read_chunked(BytesIO(data))) == b"a"
data = b"\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n"
assert b"".join(_read_chunked(BytesIO(data))) == b"ab"
data = b"\r\n"
with raises("closed prematurely"):
b"".join(_read_chunked(BytesIO(data)))
data = b"1\r\nfoo"
with raises("malformed chunked body"):
b"".join(_read_chunked(BytesIO(data)))
data = b"foo\r\nfoo"
with raises(exceptions.HttpSyntaxException):
b"".join(_read_chunked(BytesIO(data)))
data = b"5\r\naaaaa\r\n0\r\n\r\n"
with raises("too large"):
b"".join(_read_chunked(BytesIO(data), limit=2))
def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None):
"""
@type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection
@type server_conn: bool | None | mitmproxy.proxy.connection.ServerConnection
@type req: bool | None | mitmproxy.proxy.protocol.http.HTTPRequest
@type resp: bool | None | mitmproxy.proxy.protocol.http.HTTPResponse
@type err: bool | None | mitmproxy.proxy.protocol.primitives.Error
@return: mitmproxy.proxy.protocol.http.HTTPFlow
"""
if client_conn is True:
client_conn = tclient_conn()
if server_conn is True:
server_conn = tserver_conn()
if req is True:
req = tutils.treq()
if resp is True:
resp = tutils.tresp()
if err is True:
err = terr()
if req:
req = http.HTTPRequest.wrap(req)
if resp:
resp = http.HTTPResponse.wrap(resp)
f = http.HTTPFlow(client_conn, server_conn)
f.request = req
f.response = resp
f.error = err
f.reply = controller.DummyReply()
return f
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'coo=kievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_response(self, monkeypatch, logger):
logger.args = []
monkeypatch.setattr("mitmproxy.ctx.log", logger)
monkeypatch.setattr(requests, 'get', self.mocked_requests_invuln)
mocked_flow = tflow.tflow(req=tutils.treq(path=b"index.html?q=1"), resp=tutils.tresp(content=b'<html></html>'))
xss.response(mocked_flow)
assert logger.args == []
def test_host(self):
request = treq()
assert request.host == request.data.host.decode("idna")
# Test IDNA encoding
# Set str, get raw bytes
request.host = "ídna.example"
assert request.data.host == b"xn--dna-qma.example"
# Set raw bytes, get decoded
request.data.host = b"xn--idn-gla.example"
assert request.host == "idná.example"
# Set bytes, get raw bytes
request.host = b"xn--dn-qia9b.example"
assert request.data.host == b"xn--dn-qia9b.example"
# IDNA encoding is not bijective
request.host = "fußball"
assert request.host == "fussball"
# Don't fail on garbage
request.data.host = b"foo\xFF\x00bar"
assert request.host.startswith("foo")
assert request.host.endswith("bar")
# foo.bar = foo.bar should not cause any side effects.
d = request.host
request.host = d
assert request.data.host == b"foo\xFF\x00bar"
def test_response(self, get_request_invuln, logger):
mocked_flow = tflow.tflow(
req=tutils.treq(path=b"index.html?q=1"),
resp=tutils.tresp(content=b'<html></html>')
)
xss.response(mocked_flow)
assert logger.args == []
def test_get_urlencoded_form(self):
request = treq(content=b"foobar=baz")
assert not request.urlencoded_form
request.headers["Content-Type"] = "application/x-www-form-urlencoded"
assert list(request.urlencoded_form.items()) == [("foobar", "baz")]
request.raw_content = b"\xFF"
assert len(request.urlencoded_form) == 0
def post_request():
return tflow.tflow(
req=tutils.treq(
method=b'POST',
headers=(),
content=bytes(range(256))
)
)
def get_request():
return tflow.tflow(
req=tutils.treq(
method=b'GET',
content=b'',
path=b"/path?a=foo&a=bar&b=baz"
)
)
def test_set_query(self):
request = treq()
assert not request.query
request.query["foo"] = "bar"
assert request.query["foo"] == "bar"
assert request.path == "/path?foo=bar"
request.query = [('foo', 'bar')]
assert request.query["foo"] == "bar"
assert request.path == "/path?foo=bar"
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = "foo"
assert request.cookies["cookiename"] == "foo"
request.cookies = [["one", "uno"], ["two", "due"]]
assert request.cookies["one"] == "uno"
assert request.cookies["two"] == "due"
def test_read_response(input):
req = treq()
rfile = BytesIO(input)
r = read_response(rfile, req)
assert r.http_version == "HTTP/1.1"
assert r.status_code == 418
assert r.reason == "I'm a teapot"
assert r.content == b"body"
assert r.timestamp_end
def replace(self):
r = treq()
r.path = b"foobarfoo"
r.replace(b"foo", "bar")
assert r.path == b"barbarbar"
r.path = b"foobarfoo"
r.replace(b"foo", "bar", count=1)
assert r.path == b"barbarfoo"
def test_host_update_also_updates_header(self):
request = treq()
assert "host" not in request.headers
request.host = "example.com"
assert "host" not in request.headers
request.headers["Host"] = "foo"
request.host = "example.org"
assert request.headers["Host"] == "example.org"
def test_assemble_request_headers_host_header():
r = treq()
r.headers = Headers()
c = _assemble_request_headers(r.data)
assert b"host" in c
r.host = None
c = _assemble_request_headers(r.data)
assert b"host" not in c
def test_get_host_header(self):
no_hdr = treq()
assert no_hdr.host_header is None
h1 = treq(headers=(
(b"host", b"example.com"),
))
assert h1.host_header == "example.com"
h2 = treq(headers=(
(b":authority", b"example.org"),
))
assert h2.host_header == "example.org"
both_hdrs = treq(headers=(
(b"host", b"example.org"),
(b":authority", b"example.com"),
))
assert both_hdrs.host_header == "example.com"
def cycle(self, master, content):
f = tflow.tflow(req=tutils.treq(content=content))
layer = mock.Mock("mitmproxy.proxy.protocol.base.Layer")
layer.client_conn = f.client_conn
layer.reply = controller.DummyReply()
master.addons.handle_lifecycle("clientconnect", layer)
for i in eventsequence.iterate(f):
master.addons.handle_lifecycle(*i)
master.addons.handle_lifecycle("clientdisconnect", layer)
return f
def test_get_multipart_form(self):
request = treq(content=b"foobar")
assert not request.multipart_form
request.headers["Content-Type"] = "multipart/form-data"
assert list(request.multipart_form.items()) == []
with mock.patch('mitmproxy.net.http.multipart.decode') as m:
m.side_effect = ValueError
assert list(request.multipart_form.items()) == []
def test_modify_querystring(self):
m, sc = tscript("simple/modify_querystring.py")
f = tflow.tflow(req=tutils.treq(path="/search?q=term"))
m.request(f)
assert f.request.query["mitmproxy"] == "rocks"
f.request.path = "/"
m.request(f)
assert f.request.query["mitmproxy"] == "rocks"
def test_constrain_encoding(self):
request = treq()
h = request.headers.copy()
request.constrain_encoding() # no-op if there is no accept_encoding header.
assert request.headers == h
request.headers["Accept-Encoding"] = "identity, gzip, foo"
request.constrain_encoding()
assert "foo" not in request.headers["Accept-Encoding"]
assert "gzip" in request.headers["Accept-Encoding"]
def test_pretty_url(self):
request = treq()
# Without host header
assert request.url == "http://address:22/path"
assert request.pretty_url == "http://address:22/path"
# Same port as self.port (22)
request.headers["host"] = "other:22"
assert request.pretty_url == "http://other:22/path"
# Different ports
request.headers["host"] = "other"
assert request.pretty_url == "http://address:22/path"
def test_modify_querystring(self, tdata):
with taddons.context() as tctx:
sc = tctx.script(tdata.path("../examples/simple/modify_querystring.py"))
f = tflow.tflow(req=tutils.treq(path="/search?q=term"))
sc.request(f)
assert f.request.query["mitmproxy"] == "rocks"
f.request.path = "/"
sc.request(f)
assert f.request.query["mitmproxy"] == "rocks"
def flow(self, resp_content=b'message'):
times = dict(
timestamp_start=746203272,
timestamp_end=746203272,
)
# Create a dummy flow for testing
return tflow.tflow(
req=tutils.treq(method=b'GET', **times),
resp=tutils.tresp(content=resp_content, **times)
)
def test_redirect_requests(self):
m, sc = tscript("redirect_requests.py")
f = tutils.tflow(req=netutils.treq(host="example.org"))
m.request(f)
assert f.request.host == "mitmproxy.org"
def test_response(self, get_request_invuln, logger):
mocked_flow = tflow.tflow(req=tutils.treq(path=b"index.html?q=1"),
resp=tutils.tresp(content=b'<html></html>'))
xss.response(mocked_flow)
assert logger.args == []
def test_first_line_format(self):
assert treq(method=b"CONNECT").first_line_format == "authority"
assert treq(authority=b"example.com").first_line_format == "absolute"
assert treq(authority=b"").first_line_format == "relative"
def test_assemble_request_head():
c = assemble_request_head(treq(content=b"foo"))
assert b"GET" in c
assert b"qvalue" in c
assert b"content-length" in c
assert b"foo" not in c
def test_expected_http_body_size():
# Expect: 100-continue
assert expected_http_body_size(
treq(headers=Headers(expect="100-continue", content_length="42")),
expect_continue_as_0=True) == 0
# Expect: 100-continue
assert expected_http_body_size(
treq(headers=Headers(expect="100-continue", content_length="42")),
expect_continue_as_0=False) == 42
# http://tools.ietf.org/html/rfc7230#section-3.3
assert expected_http_body_size(
treq(method=b"HEAD"), tresp(headers=Headers(content_length="42"))) == 0
assert expected_http_body_size(treq(method=b"CONNECT"), tresp()) == 0
for code in (100, 204, 304):
assert expected_http_body_size(treq(), tresp(status_code=code)) == 0
# chunked
assert expected_http_body_size(
treq(headers=Headers(transfer_encoding="chunked")), ) is None
# explicit length
for val in (b"foo", b"-7"):
with pytest.raises(exceptions.HttpSyntaxException):
expected_http_body_size(treq(headers=Headers(content_length=val)))
assert expected_http_body_size(
treq(headers=Headers(content_length="42"))) == 42
# more than 1 content-length headers with same value
assert expected_http_body_size(
treq(headers=Headers([(b'content-length',
b'42'), (b'content-length', b'42')]))) == 42
# more than 1 content-length headers with conflicting value
with pytest.raises(exceptions.HttpSyntaxException):
expected_http_body_size(
treq(headers=Headers([(b'content-length',
b'42'), (b'content-length', b'45')])))
# no length
assert expected_http_body_size(treq(headers=Headers())) == 0
assert expected_http_body_size(treq(headers=Headers()),
tresp(headers=Headers())) == -1
def test_pretty_url_authority(self):
request = treq(first_line_format="authority")
assert request.pretty_url == "address:22"
def test_first_line_format(self):
_test_passthrough_attr(treq(), "first_line_format")
def test_pretty_url_authority(self):
request = treq(method=b"CONNECT", authority="address:22")
assert request.pretty_url == "address:22"
def test_pretty_url_options(self):
request = treq(method=b"OPTIONS", path=b"*")
assert request.pretty_url == "http://address:22"
def test_url_authority(self):
request = treq(method=b"CONNECT")
assert request.url == "address:22"
def test_path(self):
_test_decoded_attr(treq(), "path")
def test_port(self):
_test_passthrough_attr(treq(), "port")
def test_scheme(self):
_test_decoded_attr(treq(), "scheme")
def test_method(self):
_test_decoded_attr(treq(), "method")
def test_set_multipart_form(self):
request = treq()
request.multipart_form = [(b"file", b"shell.jpg"), (b"file_size", b"1000")]
assert request.headers["Content-Type"].startswith('multipart/form-data')
assert list(request.multipart_form.items()) == [(b"file", b"shell.jpg"), (b"file_size", b"1000")]
def test_scheme(self):
_test_decoded_attr(treq(), "scheme")
assert treq(scheme=None).scheme is None
def test_get_query(self):
request = treq()
assert not request.query
request.url = "http://localhost:80/foo?bar=42"
assert dict(request.query) == {"bar": "42"}
def test_set_multipart_form(self):
request = treq(content=b"foobar")
with pytest.raises(NotImplementedError):
request.multipart_form = "foobar"
def test_get_cookies_none(self):
request = treq()
request.headers = Headers()
assert not request.cookies
def test_send_reply_from_proxy(self):
m, sc = tscript("simple/send_reply_from_proxy.py")
f = tflow.tflow(req=tutils.treq(host="example.com", port=80))
m.request(f)
assert f.response.content == b"Hello World"
def test_get_cookies_single(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
assert len(request.cookies) == 1
assert request.cookies['cookiename'] == 'cookievalue'
def test_assemble_request_headers():
# https://github.com/mitmproxy/mitmproxy/issues/186
r = treq(content=b"")
r.headers["Transfer-Encoding"] = "chunked"
c = _assemble_request_headers(r.data)
assert b"Transfer-Encoding" in c
def test_get_path_components(self):
request = treq(path=b"/foo/bar")
assert request.path_components == ("foo", "bar")
def test_get_cookies(self):
mocked_req = tutils.treq()
mocked_req.cookies = [("cookieName2", "cookieValue2")]
mocked_flow = tflow.tflow(req=mocked_req)
# It only uses the request cookies
assert xss.get_cookies(mocked_flow) == {"cookieName2": "cookieValue2"}
def test_simple():
sio = io.StringIO()
sio_err = io.StringIO()
d = dumper.Dumper(sio, sio_err)
with taddons.context(d) as ctx:
ctx.configure(d, flow_detail=0)
d.response(tflow.tflow(resp=True))
assert not sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=1)
d.response(tflow.tflow(resp=True))
assert sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=1)
d.error(tflow.tflow(err=True))
assert sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=4)
d.response(tflow.tflow(resp=True))
assert sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=4)
d.response(tflow.tflow(resp=True))
assert "<<" in sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=4)
d.response(tflow.tflow(err=True))
assert "<<" in sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=4)
flow = tflow.tflow()
flow.request = tutils.treq()
flow.client_conn = mock.MagicMock()
flow.client_conn.address[0] = "foo"
flow.response = tutils.tresp(content=None)
flow.response.is_replay = True
flow.response.status_code = 300
d.response(flow)
assert sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=4)
flow = tflow.tflow(resp=tutils.tresp(content=b"{"))
flow.response.headers["content-type"] = "application/json"
flow.response.status_code = 400
d.response(flow)
assert sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)
ctx.configure(d, flow_detail=4)
flow = tflow.tflow()
flow.request.content = None
flow.response = http.HTTPResponse.wrap(tutils.tresp())
flow.response.content = None
d.response(flow)
assert "content missing" in sio.getvalue()
sio.truncate(0)
assert not sio_err.getvalue()
sio_err.truncate(0)