def test_assemble_request_headers():
# https://github.com/mitmproxy/mitmproxy/issues/186
r = treq(body=b"")
r.headers[b"Transfer-Encoding"] = b"chunked"
c = _assemble_request_headers(r)
assert b"Transfer-Encoding" in c
assert b"Host" in _assemble_request_headers(treq(headers=Headers()))
def test_equal(self):
a = tutils.treq(timestamp_start=42, timestamp_end=43)
b = tutils.treq(timestamp_start=42, timestamp_end=43)
assert a == b
assert not a == 'foo'
assert not b == 'foo'
assert not 'foo' == a
assert not 'foo' == b
def test_equal(self):
a = tutils.treq()
b = tutils.treq()
assert a == b
assert not a == 'foo'
assert not b == 'foo'
assert not 'foo' == a
assert not 'foo' == b
def test_equal(self):
a = tutils.treq()
b = tutils.treq()
assert a == b
assert not a == 'foo'
assert not b == 'foo'
assert not 'foo' == a
assert not 'foo' == b
def test_assemble_request():
assert assemble_request(treq()) == (b"GET /path HTTP/1.1\r\n"
b"header: qvalue\r\n"
b"content-length: 7\r\n"
b"host: address:22\r\n"
b"\r\n"
b"content")
with raises(HttpException):
assemble_request(treq(content=None))
def test_get_form_with_url_encoded(self, mock_method_urlencoded, mock_method_multipart):
req = tutils.treq()
assert req.get_form() == ODict()
req = tutils.treq()
req.body = "foobar"
req.headers["Content-Type"] = HDR_FORM_URLENCODED
req.get_form()
assert req.get_form_urlencoded.called
assert not req.get_form_multipart.called
def test_get_form_with_url_encoded(self, mock_method_urlencoded, mock_method_multipart):
req = tutils.treq()
assert req.get_form() == odict.ODict()
req = tutils.treq()
req.body = "foobar"
req.headers["Content-Type"] = [semantics.HDR_FORM_URLENCODED]
req.get_form()
assert req.get_form_urlencoded.called
assert not req.get_form_multipart.called
def test_assemble_request():
c = assemble_request(treq()) == (b"GET /path HTTP/1.1\r\n"
b"header: qvalue\r\n"
b"Host: address:22\r\n"
b"Content-Length: 7\r\n"
b"\r\n"
b"content")
with raises(HttpException):
assemble_request(treq(content=CONTENT_MISSING))
def test_expected_http_body_size():
# Expect: 100-continue
assert expected_http_body_size(
treq(headers=Headers(expect="100-continue", content_length="42"))) == 0
# http://tools.ietf.org/html/rfc7230#section-3.3
assert expected_http_body_size(
treq(method=b"HEAD"), tresp(headers=Headers(content_length="42"))) == 0
assert expected_http_body_size(treq(method=b"CONNECT"), tresp()) == 0
for code in (100, 204, 304):
assert expected_http_body_size(treq(), tresp(status_code=code)) == 0
# chunked
assert expected_http_body_size(
treq(headers=Headers(transfer_encoding="chunked")), ) is None
# explicit length
for val in (b"foo", b"-7"):
with raises(HttpSyntaxException):
expected_http_body_size(treq(headers=Headers(content_length=val)))
assert expected_http_body_size(
treq(headers=Headers(content_length="42"))) == 42
# no length
assert expected_http_body_size(treq(headers=Headers())) == 0
assert expected_http_body_size(treq(headers=Headers()),
tresp(headers=Headers())) == -1
def test_assemble_request_line():
assert _assemble_request_line(treq().data) == b"GET /path HTTP/1.1"
authority_request = treq(method=b"CONNECT", first_line_format="authority").data
assert _assemble_request_line(authority_request) == b"CONNECT address:22 HTTP/1.1"
absolute_request = treq(first_line_format="absolute").data
assert _assemble_request_line(absolute_request) == b"GET http://address:22/path HTTP/1.1"
with raises(RuntimeError):
_assemble_request_line(treq(first_line_format="invalid_form").data)
def test_assemble_request_line():
assert _assemble_request_line(treq().data) == b"GET /path HTTP/1.1"
authority_request = treq(method=b"CONNECT", first_line_format="authority").data
assert _assemble_request_line(authority_request) == b"CONNECT address:22 HTTP/1.1"
absolute_request = treq(first_line_format="absolute").data
assert _assemble_request_line(absolute_request) == b"GET http://address:22/path HTTP/1.1"
with raises(RuntimeError):
_assemble_request_line(treq(first_line_format="invalid_form").data)
def test_assemble_request():
assert assemble_request(treq()) == (
b"GET /path HTTP/1.1\r\n"
b"header: qvalue\r\n"
b"content-length: 7\r\n"
b"host: address:22\r\n"
b"\r\n"
b"content"
)
with raises(HttpException):
assemble_request(treq(content=None))
def test_assemble_request():
c = assemble_request(treq()) == (
b"GET /path HTTP/1.1\r\n"
b"header: qvalue\r\n"
b"Host: address:22\r\n"
b"Content-Length: 7\r\n"
b"\r\n"
b"content"
)
with raises(HttpException):
assemble_request(treq(content=CONTENT_MISSING))
def test_set_cookies(self):
r = tutils.treq()
r.headers = Headers(cookie="cookiename=cookievalue")
result = r.get_cookies()
result["cookiename"] = ["foo"]
r.set_cookies(result)
assert r.get_cookies()["cookiename"] == ["foo"]
def test_host(self):
request = treq()
assert request.host == request.data.host.decode("idna")
# Test IDNA encoding
# Set str, get raw bytes
request.host = "ídna.example"
assert request.data.host == b"xn--dna-qma.example"
# Set raw bytes, get decoded
request.data.host = b"xn--idn-gla.example"
assert request.host == "idná.example"
# Set bytes, get raw bytes
request.host = b"xn--dn-qia9b.example"
assert request.data.host == b"xn--dn-qia9b.example"
# IDNA encoding is not bijective
request.host = "fußball"
assert request.host == "fussball"
# Don't fail on garbage
request.data.host = b"foo\xFF\x00bar"
assert request.host.startswith("foo")
assert request.host.endswith("bar")
# foo.bar = foo.bar should not cause any side effects.
d = request.host
request.host = d
assert request.data.host == b"foo\xFF\x00bar"
def test_read_response():
req = treq()
rfile = BytesIO(b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody")
r = read_response(rfile, req)
assert r.status_code == 418
assert r.content == b"body"
assert r.timestamp_end
def test_anticache(self):
req = tutils.treq()
req.headers.add("If-Modified-Since", "foo")
req.headers.add("If-None-Match", "bar")
req.anticache()
assert "If-Modified-Since" not in req.headers
assert "If-None-Match" not in req.headers
def test_get_form_urlencoded(self):
req = tutils.treq("foobar")
assert req.get_form_urlencoded() == odict.ODict()
req.headers["Content-Type"] = semantics.HDR_FORM_URLENCODED
assert req.get_form_urlencoded() == odict.ODict(
utils.urldecode(req.body))
def test_read_response():
req = treq()
rfile = BytesIO(b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody")
r = read_response(rfile, req)
assert r.status_code == 418
assert r.content == b"body"
assert r.timestamp_end
def test_get_cookies_double(self):
r = tutils.treq()
r.headers = http.Headers(cookie="cookiename=cookievalue;othercookiename=othercookievalue")
result = r.get_cookies()
assert len(result) == 2
assert result['cookiename'] == ['cookievalue']
assert result['othercookiename'] == ['othercookievalue']
def test_legacy_first_line(self):
req = tutils.treq()
assert req.legacy_first_line('relative') == "GET /path HTTP/1.1"
assert req.legacy_first_line('authority') == "GET address:22 HTTP/1.1"
assert req.legacy_first_line('absolute') == "GET http://address:22/path HTTP/1.1"
tutils.raises(http.HttpError, req.legacy_first_line, 'foobar')
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = ["foo"]
request.cookies = result
assert request.cookies["cookiename"] == ["foo"]
def test_legacy_first_line(self):
req = tutils.treq()
assert req.legacy_first_line('relative') == "GET /path HTTP/1.1"
assert req.legacy_first_line('authority') == "GET address:22 HTTP/1.1"
assert req.legacy_first_line('absolute') == "GET http://address:22/path HTTP/1.1"
tutils.raises(http.HttpError, req.legacy_first_line, 'foobar')
def test_host(self):
if six.PY2:
from unittest import SkipTest
raise SkipTest()
request = treq()
assert request.host == request.data.host.decode("idna")
# Test IDNA encoding
# Set str, get raw bytes
request.host = "ídna.example"
assert request.data.host == b"xn--dna-qma.example"
# Set raw bytes, get decoded
request.data.host = b"xn--idn-gla.example"
assert request.host == "idná.example"
# Set bytes, get raw bytes
request.host = b"xn--dn-qia9b.example"
assert request.data.host == b"xn--dn-qia9b.example"
# IDNA encoding is not bijective
request.host = "fußball"
assert request.host == "fussball"
# Don't fail on garbage
request.data.host = b"foo\xFF\x00bar"
assert request.host.startswith("foo")
assert request.host.endswith("bar")
# foo.bar = foo.bar should not cause any side effects.
d = request.host
request.host = d
assert request.data.host == b"foo\xFF\x00bar"
def test_set_cookies(self):
r = tutils.treq()
r.headers = http.Headers(cookie="cookiename=cookievalue")
result = r.get_cookies()
result["cookiename"] = ["foo"]
r.set_cookies(result)
assert r.get_cookies()["cookiename"] == ["foo"]
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result["cookiename"] == ["coo=kievalue"]
assert result["othercookiename"] == ["othercookievalue"]
def test_anticache(self):
request = treq()
request.headers["If-Modified-Since"] = "foo"
request.headers["If-None-Match"] = "bar"
request.anticache()
assert "If-Modified-Since" not in request.headers
assert "If-None-Match" not in request.headers
def test_path(self):
req = treq()
_test_decoded_attr(req, "path")
# path can also be None.
req.path = None
assert req.path is None
assert req.data.path is None
def test_anticache(self):
req = tutils.treq()
req.headers["If-Modified-Since"] = "foo"
req.headers["If-None-Match"] = "bar"
req.anticache()
assert "If-Modified-Since" not in req.headers
assert "If-None-Match" not in req.headers
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'coo=kievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_har_extractor(self):
if sys.version_info >= (3, 0):
with tutils.raises("does not work on Python 3"):
tscript("har_extractor.py")
return
with tutils.raises(ScriptError):
tscript("har_extractor.py")
with tutils.tmpdir() as tdir:
times = dict(
timestamp_start=746203272,
timestamp_end=746203272,
)
path = os.path.join(tdir, "file")
m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path))
f = tutils.tflow(
req=netutils.treq(**times),
resp=netutils.tresp(**times)
)
self.invoke(m, "response", f)
m.addons.remove(sc)
with open(path, "rb") as f:
test_data = json.load(f)
assert len(test_data["log"]["pages"]) == 1
def test_get_cookies_double(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'cookievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_get_multipart_form(self):
request = treq(content="foobar")
assert request.multipart_form is None
request.headers["Content-Type"] = "multipart/form-data"
assert request.multipart_form == ODict(
utils.multipartdecode(request.headers, request.content))
def test_get_urlencoded_form(self):
request = treq(content="foobar")
assert request.urlencoded_form is None
request.headers["Content-Type"] = "application/x-www-form-urlencoded"
assert request.urlencoded_form == ODict(
utils.urldecode(request.content))
def test_har_extractor(self):
if sys.version_info >= (3, 0):
with tutils.raises("does not work on Python 3"):
tscript("har_extractor.py")
return
with tutils.raises(ScriptError):
tscript("har_extractor.py")
with tutils.tmpdir() as tdir:
times = dict(
timestamp_start=746203272,
timestamp_end=746203272,
)
path = os.path.join(tdir, "file")
m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path))
f = tutils.tflow(req=netutils.treq(**times),
resp=netutils.tresp(**times))
self.invoke(m, "response", f)
m.addons.remove(sc)
with open(path, "rb") as f:
test_data = json.load(f)
assert len(test_data["log"]["pages"]) == 1
def test_har_extractor(log):
if sys.version_info >= (3, 0):
with tutils.raises("does not work on Python 3"):
with example("har_extractor.py -"):
pass
return
with tutils.raises(script.ScriptException):
with example("har_extractor.py"):
pass
times = dict(
timestamp_start=746203272,
timestamp_end=746203272,
)
flow = tutils.tflow(
req=netutils.treq(**times),
resp=netutils.tresp(**times)
)
with example("har_extractor.py -") as ex:
ex.run("response", flow)
with open(tutils.test_data.path("data/har_extractor.har")) as fp:
test_data = json.load(fp)
assert json.loads(ex.ns["context"].HARLog.json()) == test_data["test_response"]
def test_get_form_with_multipart(self, mock_method_urlencoded, mock_method_multipart):
req = tutils.treq()
req.body = "foobar"
req.headers["Content-Type"] = HDR_FORM_MULTIPART
req.get_form()
assert not req.get_form_urlencoded.called
assert req.get_form_multipart.called
def test_get_form_multipart(self):
req = tutils.treq("foobar")
assert req.get_form_multipart() == odict.ODict()
req.headers["Content-Type"] = semantics.HDR_FORM_MULTIPART
assert req.get_form_multipart() == odict.ODict(
utils.multipartdecode(req.headers, req.body))
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = ["foo"]
request.cookies = result
assert request.cookies["cookiename"] == ["foo"]
def test_read_chunked():
req = treq(content=None)
req.headers["Transfer-Encoding"] = "chunked"
data = b"1\r\na\r\n0\r\n"
with raises(HttpSyntaxException):
b"".join(_read_chunked(BytesIO(data)))
data = b"1\r\na\r\n0\r\n\r\n"
assert b"".join(_read_chunked(BytesIO(data))) == b"a"
data = b"\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n"
assert b"".join(_read_chunked(BytesIO(data))) == b"ab"
data = b"\r\n"
with raises("closed prematurely"):
b"".join(_read_chunked(BytesIO(data)))
data = b"1\r\nfoo"
with raises("malformed chunked body"):
b"".join(_read_chunked(BytesIO(data)))
data = b"foo\r\nfoo"
with raises(HttpSyntaxException):
b"".join(_read_chunked(BytesIO(data)))
data = b"5\r\naaaaa\r\n0\r\n\r\n"
with raises("too large"):
b"".join(_read_chunked(BytesIO(data), limit=2))
def test_path(self):
req = treq()
_test_decoded_attr(req, "path")
# path can also be None.
req.path = None
assert req.path is None
assert req.data.path is None
def test_read_chunked():
req = treq(content=None)
req.headers["Transfer-Encoding"] = "chunked"
data = b"1\r\na\r\n0\r\n"
with raises(HttpSyntaxException):
b"".join(_read_chunked(BytesIO(data)))
data = b"1\r\na\r\n0\r\n\r\n"
assert b"".join(_read_chunked(BytesIO(data))) == b"a"
data = b"\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n"
assert b"".join(_read_chunked(BytesIO(data))) == b"ab"
data = b"\r\n"
with raises("closed prematurely"):
b"".join(_read_chunked(BytesIO(data)))
data = b"1\r\nfoo"
with raises("malformed chunked body"):
b"".join(_read_chunked(BytesIO(data)))
data = b"foo\r\nfoo"
with raises(HttpSyntaxException):
b"".join(_read_chunked(BytesIO(data)))
data = b"5\r\naaaaa\r\n0\r\n\r\n"
with raises("too large"):
b"".join(_read_chunked(BytesIO(data), limit=2))
def test_get_form_with_multipart(self, mock_method_urlencoded, mock_method_multipart):
req = tutils.treq()
req.body = "foobar"
req.headers["Content-Type"] = [semantics.HDR_FORM_MULTIPART]
req.get_form()
assert not req.get_form_urlencoded.called
assert req.get_form_multipart.called
def test_get_cookies_withequalsign(self):
r = tutils.treq()
r.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = r.get_cookies()
assert len(result) == 2
assert result['cookiename'] == ['coo=kievalue']
assert result['othercookiename'] == ['othercookievalue']
def test_get_form_for_multipart(self):
r = tutils.treq()
r.headers.add("content-type", "multipart/form-data")
r.get_form_multipart = MagicMock()
r.get_form()
assert r.get_form_multipart.called
def test_get_cookies_double(self):
request = treq()
request.headers = Headers(
cookie="cookiename=cookievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == ['cookievalue']
assert result['othercookiename'] == ['othercookievalue']
def test_get_cookies_single(self):
h = odict.ODictCaseless()
h["Cookie"] = ["cookiename=cookievalue"]
r = tutils.treq()
r.headers = h
result = r.get_cookies()
assert len(result) == 1
assert result['cookiename'] == ['cookievalue']
def test_get_cookies_single(self):
h = odict.ODictCaseless()
h["Cookie"] = ["cookiename=cookievalue"]
r = tutils.treq()
r.headers = h
result = r.get_cookies()
assert len(result) == 1
assert result["cookiename"] == ["cookievalue"]
def test_get_cookies_withequalsign(self):
r = tutils.treq()
r.headers = http.Headers(
cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = r.get_cookies()
assert len(result) == 2
assert result['cookiename'] == ['coo=kievalue']
assert result['othercookiename'] == ['othercookievalue']
def test_get_form_for_urlencoded(self):
r = tutils.treq()
r.headers.add("content-type", "application/x-www-form-urlencoded")
r.get_form_urlencoded = MagicMock()
r.get_form()
assert r.get_form_urlencoded.called
def test_read_response(input):
req = treq()
rfile = BytesIO(input)
r = read_response(rfile, req)
assert r.http_version == "HTTP/1.1"
assert r.status_code == 418
assert r.reason == "I'm a teapot"
assert r.content == b"body"
assert r.timestamp_end
def test_assemble_request_headers_host_header():
r = treq()
r.headers = Headers()
c = _assemble_request_headers(r.data)
assert b"host" in c
r.host = None
c = _assemble_request_headers(r.data)
assert b"host" not in c
def replace(self):
r = treq()
r.path = b"foobarfoo"
r.replace(b"foo", "bar")
assert r.path == b"barbarbar"
r.path = b"foobarfoo"
r.replace(b"foo", "bar", count=1)
assert r.path == b"barbarfoo"
def test_assemble_request_headers_host_header():
r = treq()
r.headers = Headers()
c = _assemble_request_headers(r.data)
assert b"host" in c
r.host = None
c = _assemble_request_headers(r.data)
assert b"host" not in c
def test_host_header_update(self):
request = treq()
assert "host" not in request.headers
request.host = "example.com"
assert "host" not in request.headers
request.headers["Host"] = "foo"
request.host = "example.org"
assert request.headers["Host"] == "example.org"
def test_modify_querystring():
flow = tutils.tflow(req=netutils.treq(path=b"/search?q=term"))
with example("modify_querystring.py") as ex:
ex.run("request", flow)
assert flow.request.query["mitmproxy"] == "rocks"
flow.request.path = "/"
ex.run("request", flow)
assert flow.request.query["mitmproxy"] == "rocks"
def test_pretty_url(self):
req = tutils.treq()
req.form_out = "authority"
assert req.pretty_url(True) == "address:22"
assert req.pretty_url(False) == "address:22"
req.form_out = "relative"
assert req.pretty_url(True) == "http://address:22/path"
assert req.pretty_url(False) == "http://address:22/path"
def flow(self, resp_content=b'message'):
times = dict(
timestamp_start=746203272,
timestamp_end=746203272,
)
# Create a dummy flow for testing
return tutils.tflow(req=netutils.treq(method=b'GET', **times),
resp=netutils.tresp(content=resp_content, **times))
def test_host_header_update(self):
request = treq()
assert "host" not in request.headers
request.host = "example.com"
assert "host" not in request.headers
request.headers["Host"] = "foo"
request.host = "example.org"
assert request.headers["Host"] == "example.org"
