def response(flow: mitmproxy.http.HTTPFlow) -> None:
if flow.response.status_code != 200:
mitmproxy.ctx.log("[-] %s" % flow.response.status_code)
if flow.response.status_code == 401:
flow.response.headers = Headers(
content_type="text/html;chartset=utf-8")
return
if flow.response.status_code == 433:
flow.response.headers = Headers(
content_type="text/html;chartset=utf-8")
flow.response.text = "<html><body><h1>403 Forbidden</h1><p>You have been blocked by Cloudflare.</p></body></html>"
return
if flow.response.status_code == 200:
body = flow.response.content.decode("utf-8")
resp = pickle.loads(b64decode(body))
r = flow.response.make(
status_code=resp.status_code,
content=b64decode(resp.data),
headers=dict(resp.headers),
)
flow.response = r
def test_echo_trailer():
sio = io.StringIO()
sio_err = io.StringIO()
d = dumper.Dumper(sio, sio_err)
with taddons.context(d) as ctx:
ctx.configure(d, flow_detail=3)
f = tflow.tflow(client_conn=True, server_conn=True, resp=True)
f.request.headers["content-type"] = "text/html"
f.request.headers["transfer-encoding"] = "chunked"
f.request.headers["trailer"] = "my-little-request-trailer"
f.request.content = b"some request content\n" * 100
f.request.trailers = Headers([(b"my-little-request-trailer",
b"foobar-request-trailer")])
f.response.headers["transfer-encoding"] = "chunked"
f.response.headers["trailer"] = "my-little-response-trailer"
f.response.content = b"some response content\n" * 100
f.response.trailers = Headers([(b"my-little-response-trailer",
b"foobar-response-trailer")])
d.echo_flow(f)
t = sio.getvalue()
assert "content-type" in t
assert "cut off" in t
assert "some request content" in t
assert "foobar-request-trailer" in t
assert "some response content" in t
assert "foobar-response-trailer" in t
def test_custom_views():
class ViewNoop(cv.View):
name = "noop"
prompt = ("noop", "n")
content_types = ["text/none"]
def __call__(self, data, **metadata):
return "noop", cv.format_text(data)
view_obj = ViewNoop()
cv.add(view_obj)
assert cv.get("noop")
r = cv.get_content_view(cv.get("noop"),
"[1, 2, 3]",
headers=Headers(content_type="text/plain"))
assert "noop" in r[0]
# now try content-type matching
r = cv.get_content_view(cv.get("Auto"),
"[1, 2, 3]",
headers=Headers(content_type="text/none"))
assert "noop" in r[0]
# now try removing the custom view
cv.remove(view_obj)
r = cv.get_content_view(cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="text/none"))
assert "noop" not in r[0]
def test_get_header_tokens():
headers = Headers()
assert get_header_tokens(headers, "foo") == []
headers["foo"] = "bar"
assert get_header_tokens(headers, "foo") == ["bar"]
headers["foo"] = "bar, voing"
assert get_header_tokens(headers, "foo") == ["bar", "voing"]
headers.set_all("foo", ["bar, voing", "oink"])
assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"]
def test_get_header_tokens():
headers = Headers()
assert get_header_tokens(headers, "foo") == []
headers["foo"] = "bar"
assert get_header_tokens(headers, "foo") == ["bar"]
headers["foo"] = "bar, voing"
assert get_header_tokens(headers, "foo") == ["bar", "voing"]
headers.set_all("foo", ["bar, voing", "oink"])
assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"]
def test_assemble_body():
c = list(assemble_body(Headers(), [b"body"]))
assert c == [b"body"]
c = list(
assemble_body(Headers(transfer_encoding="chunked"),
[b"123456789a", b""]))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
c = list(
assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"]))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
def test_expected_http_body_size():
# Expect: 100-continue
assert expected_http_body_size(
treq(headers=Headers(expect="100-continue", content_length="42"))) == 0
# http://tools.ietf.org/html/rfc7230#section-3.3
assert expected_http_body_size(
treq(method=b"HEAD"), tresp(headers=Headers(content_length="42"))) == 0
assert expected_http_body_size(treq(method=b"CONNECT"), tresp()) == 0
for code in (100, 204, 304):
assert expected_http_body_size(treq(), tresp(status_code=code)) == 0
# chunked
assert expected_http_body_size(
treq(headers=Headers(transfer_encoding="chunked")), ) is None
# explicit length
for val in (b"foo", b"-7"):
with pytest.raises(exceptions.HttpSyntaxException):
expected_http_body_size(treq(headers=Headers(content_length=val)))
assert expected_http_body_size(
treq(headers=Headers(content_length="42"))) == 42
# no length
assert expected_http_body_size(treq(headers=Headers())) == 0
assert expected_http_body_size(treq(headers=Headers()),
tresp(headers=Headers())) == -1
def test_make(self):
r = Response.make()
assert r.status_code == 200
assert r.content == b""
r = Response.make(418, "teatime")
assert r.status_code == 418
assert r.content == b"teatime"
assert r.headers["content-length"] == "7"
Response.make(content=b"foo")
Response.make(content="foo")
with pytest.raises(TypeError):
Response.make(content=42)
r = Response.make(headers=[(b"foo", b"bar")])
assert r.headers["foo"] == "bar"
r = Response.make(headers=({"foo": "baz"}))
assert r.headers["foo"] == "baz"
r = Response.make(headers=Headers(foo="qux"))
assert r.headers["foo"] == "qux"
with pytest.raises(TypeError):
Response.make(headers=42)
def test_authenticate_clean(self):
ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(),
"test")
headers = Headers()
vals = ("basic", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(
*vals)
assert ba.authenticate(headers)
ba.clean(headers)
assert ba.AUTH_HEADER not in headers
headers[ba.AUTH_HEADER] = ""
assert not ba.authenticate(headers)
headers[ba.AUTH_HEADER] = "foo"
assert not ba.authenticate(headers)
vals = ("foo", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(
*vals)
assert not ba.authenticate(headers)
ba = authentication.BasicProxyAuth(authentication.PassMan(), "test")
vals = ("basic", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(
*vals)
assert not ba.authenticate(headers)
def test_post_json(self):
p = req_post()
p.content = b'{"name": "example", "email": "*****@*****.**"}'
p.headers = Headers(content_type="application/json")
flow = tflow.tflow(req=p)
python_equals("data/test_flow_export/python_post_json.py",
export.python_code(flow))
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'coo=kievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_get_content_view():
desc, lines, err = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
)
assert "Raw" in desc
assert list(lines)
assert not err
desc, lines, err = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert desc == "JSON"
desc, lines, err = cv.get_content_view(
cv.get("JSON"),
b"[1, 2",
)
assert "Couldn't parse" in desc
with mock.patch("mitmproxy.contentviews.ViewAuto.__call__") as view_auto:
view_auto.side_effect = ValueError
desc, lines, err = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
)
assert err
assert "Couldn't parse" in desc
def test_get_cookies_simple(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=cookievalue")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
def send_error(self, flow, url, status, reason):
template_params = {}
if hasattr(flow, 'extra_data') and flow.extra_data:
template_params = flow.extra_data
template_params['url'] = url
template_params['cdx'] = {'url': url}
template_params['proxy_magic'] = self.proxy_magic
host_prefix = flow.request.req_scheme + '://' + self.proxy_magic
template_params['wbrequest'] = {'host_prefix': host_prefix}
environ = {
'pywb_proxy_magic': self.proxy_magic,
'webrec.template_params': template_params
}
msg = self.error_view.render_to_string(environ).encode('utf-8')
flow.response.content = msg
flow.response.status_code = status
flow.response.reason = reason
flow.response.headers = Headers()
flow.response.headers['Content-Type'] = 'text/html; charset=utf-8'
flow.response.headers['Content-Length'] = str(len(msg))
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
path = self.path.string()
if self.nested_response:
path += self.nested_response.parsed.spec().encode()
headers = Headers(
[header.values(settings) for header in self.headers])
body = self.body
if body:
body = body.string()
req = http.Request(
"",
0,
self.method.string(),
b'http',
b'',
path,
b"HTTP/2.0",
headers,
body,
None,
0,
0,
)
req.stream_id = self.stream_id
self.rendered_values = settings.protocol.assemble(req)
return self.rendered_values
def test_assemble_response():
assert assemble_response(tresp()) == (
b"HTTP/1.1 200 OK\r\n"
b"header-response: svalue\r\n"
b"content-length: 7\r\n"
b"\r\n"
b"message"
)
resp = tresp()
resp.headers["transfer-encoding"] = "chunked"
resp.headers["trailer"] = "my-little-trailer"
resp.trailers = Headers([(b"my-little-trailer", b"foobar")])
assert assemble_response(resp) == (
b"HTTP/1.1 200 OK\r\n"
b"header-response: svalue\r\n"
b"content-length: 7\r\n"
b"transfer-encoding: chunked\r\n"
b"trailer: my-little-trailer\r\n"
b"\r\n7\r\n"
b"message"
b"\r\n0\r\n"
b"my-little-trailer: foobar\r\n\r\n"
)
with pytest.raises(exceptions.HttpException):
assemble_response(tresp(content=None))
def test_get_cookies_no_value(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=; Expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == ""
assert len(result["cookiename"][1]) == 2
def run(self, flow: http.HTTPFlow, hdrs: Headers) -> None:
# unset all specified headers
for spec in self.replacements:
if spec.matches(flow):
hdrs.pop(spec.subject, None)
# set all specified headers if the replacement string is not empty
for spec in self.replacements:
if spec.matches(flow):
try:
replacement = spec.read_replacement()
except OSError as e:
ctx.log.warn(f"Could not read replacement file: {e}")
continue
else:
if replacement:
hdrs.add(spec.subject, replacement)
def test_client_handshake_headers(_):
assert websocket_utils.client_handshake_headers() == \
Headers([
(b'connection', b'upgrade'),
(b'upgrade', b'websocket'),
(b'sec-websocket-version', b'13'),
(b'sec-websocket-key', b'cHVtcGtpbnNwdW1wa2lucw=='),
])
assert websocket_utils.client_handshake_headers(b"13", b"foobar", b"foo", b"bar") == \
Headers([
(b'connection', b'upgrade'),
(b'upgrade', b'websocket'),
(b'sec-websocket-version', b'13'),
(b'sec-websocket-key', b'foobar'),
(b'sec-websocket-protocol', b'foo'),
(b'sec-websocket-extensions', b'bar')
])
def test_encode():
data = [(b"file", b"shell.jpg"), (b"file_size", b"1000")]
headers = Headers(
content_type='multipart/form-data; boundary=127824672498')
content = multipart.encode(headers, data)
assert b'Content-Disposition: form-data; name="file"' in content
assert b'Content-Type: text/plain; charset=utf-8\r\n\r\nshell.jpg\r\n\r\n--127824672498\r\n' in content
assert b'1000\r\n\r\n--127824672498--\r\n'
assert len(content) == 252
with pytest.raises(ValueError, match=r"boundary found in encoded string"):
multipart.encode(headers, [(b"key", b"--127824672498")])
boundary = 'boundary茅莽'
headers = Headers(content_type='multipart/form-data; boundary=' + boundary)
result = multipart.encode(headers, data)
assert result == b''
def test_server_handshake_headers():
assert websocket_utils.server_handshake_headers("foobar", "foo", "bar") == \
Headers([
(b'connection', b'upgrade'),
(b'upgrade', b'websocket'),
(b'sec-websocket-accept', b'AzhRPA4TNwR6I/riJheN0TfR7+I='),
(b'sec-websocket-protocol', b'foo'),
(b'sec-websocket-extensions', b'bar'),
])
def test_assemble_request_headers_host_header():
r = treq()
r.headers = Headers()
c = _assemble_request_headers(r.data)
assert b"host" in c
r.host = None
c = _assemble_request_headers(r.data)
assert b"host" not in c
def custom_header(host, content_type, size):
header = Headers(host=host)
header['Pragma'] = 'no-store, max-age=0'
header['Cache-Control'] = 'no-store, max-age=0'
header["Connection"] = "close"
header["Server"] = "Smarthost/1.0.0.3 for mitmproxy/2.0.2 - by mooring"
header["Content-Length"] = str(size)
if content_type is not None: header['Content-type'] = content_type
return header
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = "foo"
assert request.cookies["cookiename"] == "foo"
request.cookies = [["one", "uno"], ["two", "due"]]
assert request.cookies["one"] == "uno"
assert request.cookies["two"] == "due"
def test_assemble_body():
c = list(assemble_body(Headers(), [b"body"], Headers()))
assert c == [b"body"]
c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""], Headers()))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers()))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers(trailer="trailer")))
assert c == [b"a\r\n123456789a\r\n", b"0\r\ntrailer: trailer\r\n\r\n"]
with pytest.raises(exceptions.HttpException):
list(assemble_body(Headers(), [b"body"], Headers(trailer="trailer")))
def response(flow: http.HTTPFlow):
if flow.response.trailers:
print("HTTP Trailers detected! Response contains:",
flow.response.trailers)
if flow.request.path == "/inject_trailers":
flow.response.headers["trailer"] = "x-my-injected-trailer-header"
flow.response.trailers = Headers([(b"x-my-injected-trailer-header",
b"foobar")])
print("Injected a new trailer...", flow.response.headers["trailer"])
def test_get_cookies_twocookies(self):
resp = tresp()
resp.headers = Headers([[b"Set-Cookie", b"cookiename=cookievalue"],
[b"Set-Cookie", b"othercookie=othervalue"]])
result = resp.cookies
assert len(result) == 2
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
assert "othercookie" in result
assert result["othercookie"] == ("othervalue", CookieAttrs())
def test_view_multipart(self):
view = cv.ViewMultipart()
v = b"""
--AaB03x
Content-Disposition: form-data; name="submit-name"
Larry
--AaB03x
""".strip()
h = Headers(content_type="multipart/form-data; boundary=AaB03x")
assert view(v, headers=h)
h = Headers()
assert not view(v, headers=h)
h = Headers(content_type="multipart/form-data")
assert not view(v, headers=h)
h = Headers(content_type="unparseable")
assert not view(v, headers=h)
def test_modify_form(self):
m, sc = tscript("simple/modify_form.py")
form_header = Headers(content_type="application/x-www-form-urlencoded")
f = tflow.tflow(req=tutils.treq(headers=form_header))
m.request(f)
assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks"
f.request.headers["content-type"] = ""
m.request(f)
assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")]
def test_get_cookies_with_parameters(self):
resp = tresp()
cookie = "cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly"
resp.headers = Headers(set_cookie=cookie)
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == "cookievalue"
attrs = result["cookiename"][1]
assert len(attrs) == 4
assert attrs["domain"] == "example.com"
assert attrs["expires"] == "Wed Oct 21 16:29:41 2015"
assert attrs["path"] == "/"
assert attrs["httponly"] == ""
def test_connection_close():
headers = Headers()
assert connection_close(b"HTTP/1.0", headers)
assert not connection_close(b"HTTP/1.1", headers)
headers["connection"] = "keep-alive"
assert not connection_close(b"HTTP/1.1", headers)
headers["connection"] = "close"
assert connection_close(b"HTTP/1.1", headers)
headers["connection"] = "foobar"
assert connection_close(b"HTTP/1.0", headers)
assert not connection_close(b"HTTP/1.1", headers)