def test_replace_simple(self):
headers = Headers(Host="example.com", Accept="text/plain")
replacements = headers.replace("Host: ", "X-Host: ")
assert replacements == 1
assert headers["X-Host"] == "example.com"
assert "Host" not in headers
assert headers["Accept"] == "text/plain"
def test_expected_http_body_size():
# Expect: 100-continue
assert expected_http_body_size(
treq(headers=Headers(expect="100-continue", content_length="42"))) == 0
# http://tools.ietf.org/html/rfc7230#section-3.3
assert expected_http_body_size(
treq(method=b"HEAD"), tresp(headers=Headers(content_length="42"))) == 0
assert expected_http_body_size(treq(method=b"CONNECT"), tresp()) == 0
for code in (100, 204, 304):
assert expected_http_body_size(treq(), tresp(status_code=code)) == 0
# chunked
assert expected_http_body_size(
treq(headers=Headers(transfer_encoding="chunked")), ) is None
# explicit length
for val in (b"foo", b"-7"):
with raises(HttpSyntaxException):
expected_http_body_size(treq(headers=Headers(content_length=val)))
assert expected_http_body_size(
treq(headers=Headers(content_length="42"))) == 42
# no length
assert expected_http_body_size(treq()) == 0
assert expected_http_body_size(treq(), tresp()) == -1
def test_replace_simple(self):
headers = Headers(Host="example.com", Accept="text/plain")
replacements = headers.replace("Host: ", "X-Host: ")
assert replacements == 1
assert headers["X-Host"] == "example.com"
assert "Host" not in headers
assert headers["Accept"] == "text/plain"
def test_view_auto(self):
v = cv.ViewAuto()
f = v(
"foo",
headers=Headers()
)
assert f[0] == "Raw"
f = v(
"<html></html>",
headers=Headers(content_type="text/html")
)
assert f[0] == "HTML"
f = v(
"foo",
headers=Headers(content_type="text/flibble")
)
assert f[0] == "Raw"
f = v(
"<xml></xml>",
headers=Headers(content_type="text/flibble")
)
assert f[0].startswith("XML")
def test_get_content_view(self):
r = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json"))
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json"))
assert r[0] == "JSON"
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
headers=Headers(content_type="application/json"))
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/vnd.api+json"))
assert r[0] == "JSON"
tutils.raises(ContentViewException,
cv.get_content_view,
cv.get("AMF"),
b"[1, 2",
headers=Headers())
def test_custom_views():
class ViewNoop(cv.View):
name = "noop"
prompt = ("noop", "n")
content_types = ["text/none"]
def __call__(self, data, **metadata):
return "noop", cv.format_text(data)
view_obj = ViewNoop()
cv.add(view_obj)
assert cv.get("noop")
r = cv.get_content_view(cv.get("noop"),
"[1, 2, 3]",
headers=Headers(content_type="text/plain"))
assert "noop" in r[0]
# now try content-type matching
r = cv.get_content_view(cv.get("Auto"),
"[1, 2, 3]",
headers=Headers(content_type="text/none"))
assert "noop" in r[0]
# now try removing the custom view
cv.remove(view_obj)
r = cv.get_content_view(cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="text/none"))
assert "noop" not in r[0]
def test_keys(self):
headers = Headers(Host="example.com")
assert len(headers.keys()) == 1
assert headers.keys()[0] == "Host"
headers = self._2host()
assert len(headers.keys()) == 1
assert headers.keys()[0] == "Host"
def test_assemble_body():
c = list(assemble_body(Headers(), [b"body"]))
assert c == [b"body"]
c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""]))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
c = list(assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"]))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
def test_get_header_tokens():
headers = Headers()
assert get_header_tokens(headers, "foo") == []
headers["foo"] = "bar"
assert get_header_tokens(headers, "foo") == ["bar"]
headers["foo"] = "bar, voing"
assert get_header_tokens(headers, "foo") == ["bar", "voing"]
headers.set_all("foo", ["bar, voing", "oink"])
assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"]
def test_state(self):
headers = self._2host()
assert len(headers.get_state()) == 2
assert headers == Headers.from_state(headers.get_state())
headers2 = Headers()
assert headers != headers2
headers2.load_state(headers.get_state())
assert headers == headers2
def test_state(self):
headers = self._2host()
assert len(headers.get_state()) == 2
assert headers == Headers.from_state(headers.get_state())
headers2 = Headers()
assert headers != headers2
headers2.load_state(headers.get_state())
assert headers == headers2
def test_get_header_tokens():
headers = Headers()
assert utils.get_header_tokens(headers, "foo") == []
headers["foo"] = "bar"
assert utils.get_header_tokens(headers, "foo") == ["bar"]
headers["foo"] = "bar, voing"
assert utils.get_header_tokens(headers, "foo") == ["bar", "voing"]
headers.set_all("foo", ["bar, voing", "oink"])
assert utils.get_header_tokens(headers, "foo") == ["bar", "voing", "oink"]
def test_read_http_body():
# test default case
headers = Headers()
headers["content-length"] = "7"
data = "testing"
assert mock_protocol(data).read_http_body(headers, None, "GET", 200, False) == "testing"
# test content length: invalid header
headers["content-length"] = "foo"
data = "testing"
tutils.raises(
http.HttpError,
mock_protocol(data).read_http_body,
headers, None, "GET", 200, False
)
# test content length: invalid header #2
headers["content-length"] = "-1"
data = "testing"
tutils.raises(
http.HttpError,
mock_protocol(data).read_http_body,
headers, None, "GET", 200, False
)
# test content length: content length > actual content
headers["content-length"] = "5"
data = "testing"
tutils.raises(
http.HttpError,
mock_protocol(data).read_http_body,
headers, 4, "GET", 200, False
)
# test content length: content length < actual content
data = "testing"
assert len(mock_protocol(data).read_http_body(headers, None, "GET", 200, False)) == 5
# test no content length: limit > actual content
headers = Headers()
data = "testing"
assert len(mock_protocol(data).read_http_body(headers, 100, "GET", 200, False)) == 7
# test no content length: limit < actual content
data = "testing"
tutils.raises(
http.HttpError,
mock_protocol(data).read_http_body,
headers, 4, "GET", 200, False
)
# test chunked
headers = Headers()
headers["transfer-encoding"] = "chunked"
data = "5\r\naaaaa\r\n0\r\n\r\n"
assert mock_protocol(data).read_http_body(headers, 100, "GET", 200, False) == "aaaaa"
def test_str(self):
headers = Headers(Host="example.com")
assert bytes(headers) == b"Host: example.com\r\n"
headers = Headers([[b"Host", b"example.com"],
[b"Accept", b"text/plain"]])
assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n"
headers = Headers()
assert bytes(headers) == b""
def test_eq_ne(self):
headers1 = Headers(Host="example.com")
headers2 = Headers(host="example.com")
assert not (headers1 == headers2)
assert headers1 != headers2
headers1 = Headers(Host="example.com")
headers2 = Headers(Host="example.com")
assert headers1 == headers2
assert not (headers1 != headers2)
assert headers1 != 42
def req(self):
headers = Headers(header="qvalue")
req = http.HTTPRequest("absolute", "GET", "http", "host", 80, "/path",
(1, 1), headers, "content_request", None, None)
f = http.HTTPFlow(tutils.tclient_conn(), None)
f.request = req
return f
def make_error_response(status_code, message, headers=None):
response = status_codes.RESPONSES.get(status_code, "Unknown")
body = """
<html>
<head>
<title>%d %s</title>
</head>
<body>%s</body>
</html>
""".strip() % (status_code, response, message)
if not headers:
headers = Headers(
Server=version.NAMEVERSION,
Connection="close",
Content_Length=str(len(body)),
Content_Type="text/html"
)
return HTTPResponse(
b"HTTP/1.1",
status_code,
response,
headers,
body,
)
def block_request(flow):
"""
Modifies flow and blocks request for suspected proxy usage
"""
resp = HTTPResponse("HTTP/1.1", 200, "OK",
Headers(Context_Type="text/html"), block_html)
flow.reply(resp)
def make_error_response(status_code, message, headers=None):
response = status_codes.RESPONSES.get(status_code, "Unknown").encode()
body = """
<html>
<head>
<title>%d %s</title>
</head>
<body>%s</body>
</html>
""".strip() % (status_code, response, cgi.escape(message))
body = body.encode("utf8", "replace")
if not headers:
headers = Headers(Server=version.MITMPROXY,
Connection="close",
Content_Length=str(len(body)),
Content_Type="text/html")
return HTTPResponse(
b"HTTP/1.1",
status_code,
response,
headers,
body,
)
def test_simple_message(self):
data = """
HTTP/1.1 200 OK
"""
assert self.tst(data, "GET", None) == http.Response(
(1, 1), 200, 'OK', Headers(), ''
)
def test_expected_http_body_size():
# gibber in the content-length field
headers = Headers(content_length="foo")
assert HTTP1Protocol.expected_http_body_size(headers, False, "GET", 200) is None
# negative number in the content-length field
headers = Headers(content_length="-7")
assert HTTP1Protocol.expected_http_body_size(headers, False, "GET", 200) is None
# explicit length
headers = Headers(content_length="5")
assert HTTP1Protocol.expected_http_body_size(headers, False, "GET", 200) == 5
# no length
headers = Headers()
assert HTTP1Protocol.expected_http_body_size(headers, False, "GET", 200) == -1
# no length request
headers = Headers()
assert HTTP1Protocol.expected_http_body_size(headers, True, "GET", None) == 0
def test_post_json(self):
req_post.content = '{"name": "example", "email": "*****@*****.**"}'
req_post.headers = Headers(content_type="application/json")
flow = tutils.tflow(req=req_post)
result = dedent("""
import requests
url = 'http://address/path'
headers = {
'content-type': 'application/json',
}
json = {
"name": "example",
"email": "*****@*****.**"
}
response = requests.request(
method='POST',
url=url,
headers=headers,
json=json,
)
print(response.text)
""").strip()
assert flow_export.python_code(flow) == result
def _restore(flow, rawhead):
# Building usable headers
headers = Headers()
lines = rawhead.decode('utf-8')[:-2].split("\r\n")
for line in lines:
temp = line.split(": ")
headers[temp[0]] = temp[1]
body = cache.get('http.cache.body.%s' % flow.request.pretty_url)
if len(body) == 0:
print("Cache hit but body empty, let's doing a real request")
cache.delete('http.cache.body.%s' % flow.request.pretty_url)
cache.delete('http.cache.head.%s' % flow.request.pretty_url)
return
# Building response from cache
response = HTTPResponse(b"HTTP/1.1", 200, b"OK", headers, body)
print(response)
response.headers["X-GIG-Cache"] = "from-cache"
# Send forged response
flow.reply.send(response)
def _2host(self):
return Headers(
[
[b"Host", b"example.com"],
[b"host", b"example.org"]
]
)
def request(context, flow):
rewrite = False
mockresp = open("mock/success.json", "r").read()
flow.request.oldpath = flow.request.path
# if flow.request.path.endswith("/api/2/account/api/"):
# flow.request.path = "/api/2/account/myapitest"
if flow.request.path.startswith(
"/api/2/account/my/profile/plan/circlesswitch/upgrade/"):
rewrite = True
if flow.request.path.startswith(
"/api/2/account/my/profile/id/digits/verify/get/"):
mockresp = open("mock/my_profile_id_digits_verify_get.json",
"r").read()
rewrite = True
if flow.request.path.startswith(
"/api/2/account/my/portin/request/cancel/"):
rewrite = True
print("\nRequest : " + str(flow.request.oldpath) + "\n")
# print("Request Rewrite : " + str(flow.request.path)+"\n")
print("\nRequest Body : " + str(flow.request.content) + "\n")
if rewrite:
time.sleep(0.3)
resp = HTTPResponse(
"HTTP/1.1", 200, "OK",
Headers(Content_Type="application/json; charset=utf-8"), mockresp)
flow.reply(resp)
print("\nResponse : " + str(flow.response.content) + "\n")
def inline_flash(self, swf):
content = base64.standard_b64encode(swf)
return HTTPResponse(
"HTTP/1.1", 200, "OK",
Headers(content_type="text/html", content_length=str(len(content))),
content)
def test_post_json(self):
p = req_post()
p.content = '{"name": "example", "email": "*****@*****.**"}'
p.headers = Headers(content_type="application/json")
flow = tutils.tflow(req=p)
python_equals("test_flow_export/python_post_json.py",
flow_export.python_code(flow))
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
path = self.path.string()
if self.nested_response:
path += self.nested_response.parsed.spec().encode()
headers = Headers(
[header.values(settings) for header in self.headers])
body = self.body
if body:
body = body.string()
req = http.Request(
b'',
self.method.string(),
b'',
b'',
b'',
path,
(2, 0),
headers,
body,
)
req.stream_id = self.stream_id
self.rendered_values = settings.protocol.assemble(req)
return self.rendered_values
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'coo=kievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_authenticate_clean(self):
ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(),
"test")
headers = Headers()
vals = ("basic", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(
*vals)
assert ba.authenticate(headers)
ba.clean(headers)
assert ba.AUTH_HEADER not in headers
headers[ba.AUTH_HEADER] = ""
assert not ba.authenticate(headers)
headers[ba.AUTH_HEADER] = "foo"
assert not ba.authenticate(headers)
vals = ("foo", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(
*vals)
assert not ba.authenticate(headers)
ba = authentication.BasicProxyAuth(authentication.PassMan(), "test")
vals = ("basic", "foo", "bar")
headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(
*vals)
assert not ba.authenticate(headers)
def test_get_cookies_simple(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=cookievalue")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
def create_request(self, method, scheme, host, port, path):
"""
this method creates a new artificial and minimalist request also adds it to flowlist
"""
c = ClientConnection.make_dummy(("", 0))
s = ServerConnection.make_dummy((host, port))
s = ServerConnection.from_state(
dict(address=dict(address=(host, port), use_ipv6=False),
ip_address=None,
cert=None,
sni=host,
source_address=dict(address=("", 0), use_ipv6=False),
ssl_established=True,
timestamp_start=None,
timestamp_tcp_setup=None,
timestamp_ssl_setup=None,
timestamp_end=None,
via=None))
f = HTTPFlow(c, s)
headers = Headers()
req = HTTPRequest("absolute", method, scheme, host, port, path,
b"HTTP/1.1", headers, b"")
f.request = req
return self.load_flow(f)
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = ["foo"]
request.cookies = result
assert request.cookies["cookiename"] == ["foo"]
def test_get_content_view():
desc, lines, err = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
)
assert "Raw" in desc
assert list(lines)
assert not err
desc, lines, err = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json"))
assert desc == "JSON"
desc, lines, err = cv.get_content_view(
cv.get("JSON"),
b"[1, 2",
)
assert "Couldn't parse" in desc
with mock.patch("mitmproxy.contentviews.ViewAuto.__call__") as view_auto:
view_auto.side_effect = ValueError
desc, lines, err = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
)
assert err
assert "Couldn't parse" in desc
def test_set_all(self):
headers = Headers(Host="example.com")
headers.set_all("Accept", ["text/plain"])
assert len(headers) == 2
assert "accept" in headers
headers = self._2host()
headers.set_all("Host", ["example.org"])
assert headers["host"] == "example.org"
headers.set_all("Host", ["example.org", "example.net"])
assert headers["host"] == "example.org, example.net"
def test_keys(self):
headers = Headers(Host="example.com")
assert list(headers.keys()) == ["Host"]
headers = self._2host()
assert list(headers.keys()) == ["Host"]
def test_replace_remove_spacer(self):
headers = Headers(Host="example.com")
replacements = headers.replace(r"Host: ", "X-Host ")
assert replacements == 0
assert headers["Host"] == "example.com"