def test_echo_trailer():
sio = io.StringIO()
sio_err = io.StringIO()
d = dumper.Dumper(sio, sio_err)
with taddons.context(d) as ctx:
ctx.configure(d, flow_detail=3)
f = tflow.tflow(client_conn=True, server_conn=True, resp=True)
f.request.headers["content-type"] = "text/html"
f.request.headers["transfer-encoding"] = "chunked"
f.request.headers["trailer"] = "my-little-request-trailer"
f.request.content = b"some request content\n" * 100
f.request.trailers = Headers([(b"my-little-request-trailer", b"foobar-request-trailer")])
f.response.headers["transfer-encoding"] = "chunked"
f.response.headers["trailer"] = "my-little-response-trailer"
f.response.content = b"some response content\n" * 100
f.response.trailers = Headers([(b"my-little-response-trailer", b"foobar-response-trailer")])
d.echo_flow(f)
t = sio.getvalue()
assert "content-type" in t
assert "cut off" in t
assert "some request content" in t
assert "foobar-request-trailer" in t
assert "some response content" in t
assert "foobar-response-trailer" in t
def test_insert(self):
headers = Headers(Accept="text/plain")
headers.insert(0, b"Host", "example.com")
assert headers.fields == (
(b'Host', b'example.com'),
(b'Accept', b'text/plain')
)
def test_get_header_tokens():
headers = Headers()
assert get_header_tokens(headers, "foo") == []
headers["foo"] = "bar"
assert get_header_tokens(headers, "foo") == ["bar"]
headers["foo"] = "bar, voing"
assert get_header_tokens(headers, "foo") == ["bar", "voing"]
headers.set_all("foo", ["bar, voing", "oink"])
assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"]
def test_bytes(self):
headers = Headers(Host="example.com")
assert bytes(headers) == b"Host: example.com\r\n"
headers = Headers([(b"Host", b"example.com"),
(b"Accept", b"text/plain")])
assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n"
headers = Headers()
assert bytes(headers) == b""
def test_validate_headers():
# both content-length and chunked (possible request smuggling)
with pytest.raises(
ValueError,
match=
"Received both a Transfer-Encoding and a Content-Length header"):
validate_headers(
Headers(transfer_encoding="chunked", content_length="42"), )
with pytest.raises(ValueError, match="Received an invalid header name"):
validate_headers(Headers([(b"content-length ", b"42")]), )
def test_items(self):
headers = Headers([
(b"Set-Cookie", b"foo"),
(b"Set-Cookie", b"bar"),
(b'Accept', b'text/plain'),
])
assert list(headers.items()) == [('Set-Cookie', 'foo, bar'),
('Accept', 'text/plain')]
assert list(headers.items(multi=True)) == [('Set-Cookie', 'foo'),
('Set-Cookie', 'bar'),
('Accept', 'text/plain')]
def test_make(self):
r = Request.make("GET", "https://example.com/")
assert r.method == "GET"
assert r.scheme == "https"
assert r.host == "example.com"
assert r.port == 443
assert r.path == "/"
r = Request.make("GET", "https://example.com/", "content", {"Foo": "bar"})
assert r.content == b"content"
assert r.headers["content-length"] == "7"
assert r.headers["Foo"] == "bar"
Request.make("GET", "https://example.com/", content=b"content")
with pytest.raises(TypeError):
Request.make("GET", "https://example.com/", content=42)
r = Request.make("GET", "https://example.com/", headers=[(b"foo", b"bar")])
assert r.headers["foo"] == "bar"
r = Request.make("GET", "https://example.com/", headers=({"foo": "baz"}))
assert r.headers["foo"] == "baz"
r = Request.make("GET", "https://example.com/", headers=Headers(foo="qux"))
assert r.headers["foo"] == "qux"
with pytest.raises(TypeError):
Request.make("GET", "https://example.com/", headers=42)
def _read_headers(lines: Iterable[bytes]) -> Headers:
"""
Read a set of headers.
Stop once a blank line is reached.
Returns:
A headers object
Raises:
exceptions.HttpSyntaxException
"""
ret: List[Tuple[bytes, bytes]] = []
for line in lines:
if line[0] in b" \t":
if not ret:
raise ValueError("Invalid headers")
# continued header
ret[-1] = (ret[-1][0], ret[-1][1] + b'\r\n ' + line.strip())
else:
try:
name, value = line.split(b":", 1)
value = value.strip()
if not name:
raise ValueError()
ret.append((name, value))
except ValueError:
raise ValueError(f"Invalid header line: {line!r}")
return Headers(ret)
def test_serializable(self):
data1 = tresp(timestamp_start=42, timestamp_end=42).data
data1.trailers = Headers()
data2 = tresp().data.from_state(
data1.get_state()) # ResponseData.from_state()
assert data1 == data2
def test_content_length_not_added_for_response_with_transfer_encoding(
self):
headers = Headers(((b"transfer-encoding", b"chunked"), ))
resp = tresp(headers=headers)
resp.content = b"bar"
assert "content-length" not in resp.headers
def test_get_cookies_withequalsign(self):
request = treq()
request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
result = request.cookies
assert len(result) == 2
assert result['cookiename'] == 'coo=kievalue'
assert result['othercookiename'] == 'othercookievalue'
def test_make(self):
r = Response.make()
assert r.status_code == 200
assert r.content == b""
r = Response.make(418, "teatime")
assert r.status_code == 418
assert r.content == b"teatime"
assert r.headers["content-length"] == "7"
Response.make(content=b"foo")
Response.make(content="foo")
with pytest.raises(TypeError):
Response.make(content=42)
r = Response.make(headers=[(b"foo", b"bar")])
assert r.headers["foo"] == "bar"
r = Response.make(headers=({"foo": "baz"}))
assert r.headers["foo"] == "baz"
r = Response.make(headers=Headers(foo="qux"))
assert r.headers["foo"] == "qux"
with pytest.raises(TypeError):
Response.make(headers=42)
def test_get_cookies_simple(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=cookievalue")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
def _2host(self):
return Headers(
(
(b"Host", b"example.com"),
(b"host", b"example.org")
)
)
def test_set(self):
headers = Headers()
headers["foo"] = "1"
headers[b"bar"] = b"2"
headers["baz"] = b"3"
with pytest.raises(TypeError):
headers["foobar"] = 42
assert len(headers) == 3
def run(self, flow: http.HTTPFlow, hdrs: Headers) -> None:
# unset all specified headers
for spec in self.replacements:
if spec.matches(flow):
hdrs.pop(spec.subject, None)
# set all specified headers if the replacement string is not empty
for spec in self.replacements:
if spec.matches(flow):
try:
replacement = spec.read_replacement()
except OSError as e:
ctx.log.warn(f"Could not read replacement file: {e}")
continue
else:
if replacement:
hdrs.add(spec.subject, replacement)
def test_get_cookies_no_value(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=; Expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == ""
assert len(result["cookiename"][1]) == 2
def __convert_headers_for_mitm(self, headers):
headers_obj = json.loads(headers)
headers_list = []
for key, value in headers_obj.items():
header = (key.encode(), value.encode())
headers_list.append(header)
return Headers(headers_list)
def test_set_cookies(self):
request = treq()
request.headers = Headers(cookie="cookiename=cookievalue")
result = request.cookies
result["cookiename"] = "foo"
assert request.cookies["cookiename"] == "foo"
request.cookies = [["one", "uno"], ["two", "due"]]
assert request.cookies["one"] == "uno"
assert request.cookies["two"] == "due"
def test_encode():
data = [(b"file", b"shell.jpg"), (b"file_size", b"1000")]
headers = Headers(
content_type='multipart/form-data; boundary=127824672498')
content = multipart.encode(headers, data)
assert b'Content-Disposition: form-data; name="file"' in content
assert b'Content-Type: text/plain; charset=utf-8\r\n\r\nshell.jpg\r\n\r\n--127824672498\r\n' in content
assert b'1000\r\n\r\n--127824672498--\r\n'
assert len(content) == 252
with pytest.raises(ValueError, match=r"boundary found in encoded string"):
multipart.encode(headers, [(b"key", b"--127824672498")])
boundary = 'boundary茅莽'
headers = Headers(content_type='multipart/form-data; boundary=' + boundary)
result = multipart.encode(headers, data)
assert result == b''
def test_get_cookies_twocookies(self):
resp = tresp()
resp.headers = Headers([[b"Set-Cookie", b"cookiename=cookievalue"],
[b"Set-Cookie", b"othercookie=othervalue"]])
result = resp.cookies
assert len(result) == 2
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
assert "othercookie" in result
assert result["othercookie"] == ("othervalue", CookieAttrs())
def test_generate_tflow_js(tdata):
tf_http = tflow.tflow(resp=True, err=True, ws=True)
tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29"
tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939"
tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8"
tf_http.server_conn.certificate_list = [
certs.Cert.from_pem(
Path(
tdata.path(
"mitmproxy/net/data/verificationcerts/self-signed.pem")).
read_bytes())
]
tf_http.request.trailers = Headers(trailer="qvalue")
tf_http.response.trailers = Headers(trailer="qvalue")
tf_http.comment = "I'm a comment!"
tf_tcp = tflow.ttcpflow(err=True)
tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001"
tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322"
tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200"
# language=TypeScript
content = (
"/** Auto-generated by test_app.py:test_generate_tflow_js */\n"
"import {HTTPFlow, TCPFlow} from '../../flow';\n"
"export function THTTPFlow(): Required<HTTPFlow> {\n"
" return %s\n"
"}\n"
"export function TTCPFlow(): Required<TCPFlow> {\n"
" return %s\n"
"}" % (
textwrap.indent(
json.dumps(app.flow_to_json(tf_http), indent=4,
sort_keys=True), " "),
textwrap.indent(
json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True),
" "),
))
content = content.replace(": null", ": undefined")
(Path(__file__).parent /
"../../../../web/src/js/__tests__/ducks/_tflow.ts").write_bytes(
content.encode())
def test_assemble_body():
c = list(assemble_body(Headers(), [b"body"], Headers()))
assert c == [b"body"]
c = list(
assemble_body(Headers(transfer_encoding="chunked"),
[b"123456789a", b""], Headers()))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
c = list(
assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"],
Headers()))
assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"]
c = list(
assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"],
Headers(trailer="trailer")))
assert c == [b"a\r\n123456789a\r\n", b"0\r\ntrailer: trailer\r\n\r\n"]
with pytest.raises(ValueError):
list(assemble_body(Headers(), [b"body"], Headers(trailer="trailer")))
def test_init(self):
headers = Headers()
assert len(headers) == 0
headers = Headers([(b"Host", b"example.com")])
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(Host="example.com")
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[(b"Host", b"invalid")],
Host="example.com"
)
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[(b"Host", b"invalid"), (b"Accept", b"text/plain")],
Host="example.com"
)
assert len(headers) == 2
assert headers["Host"] == "example.com"
assert headers["Accept"] == "text/plain"
with pytest.raises(TypeError):
Headers([(b"Host", "not-bytes")])
def test_modify_form(self, tdata):
with taddons.context() as tctx:
sc = tctx.script(tdata.path("../examples/addons/http-modify-form.py"))
form_header = Headers(content_type="application/x-www-form-urlencoded")
f = tflow.tflow(req=tutils.treq(headers=form_header))
sc.request(f)
assert f.request.urlencoded_form["mitmproxy"] == "rocks"
f.request.headers["content-type"] = ""
sc.request(f)
assert list(f.request.urlencoded_form.items()) == [("foo", "bar")]
def test_get_cookies_with_parameters(self):
resp = tresp()
cookie = "cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly"
resp.headers = Headers(set_cookie=cookie)
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == "cookievalue"
attrs = result["cookiename"][1]
assert len(attrs) == 4
assert attrs["domain"] == "example.com"
assert attrs["expires"] == "Wed Oct 21 16:29:41 2015"
assert attrs["path"] == "/"
assert attrs["httponly"] == ""
def run(self, flow: http.HTTPFlow, hdrs: Headers) -> None:
matches = []
# first check all the filters against the original, unmodified flow
for spec in self.replacements:
matches.append(spec.matches(flow))
# unset all specified headers
for i, spec in enumerate(self.replacements):
if matches[i]:
hdrs.pop(spec.subject, None)
# set all specified headers if the replacement string is not empty
for i, spec in enumerate(self.replacements):
if matches[i]:
try:
replacement = spec.read_replacement()
except OSError as e:
ctx.log.warn(f"Could not read replacement file: {e}")
continue
else:
if replacement:
hdrs.add(spec.subject, replacement)
def test_connection_close():
headers = Headers()
assert connection_close(b"HTTP/1.0", headers)
assert not connection_close(b"HTTP/1.1", headers)
assert not connection_close(b"HTTP/2.0", headers)
headers["connection"] = "keep-alive"
assert not connection_close(b"HTTP/1.1", headers)
headers["connection"] = "close"
assert connection_close(b"HTTP/1.1", headers)
headers["connection"] = "foobar"
assert connection_close(b"HTTP/1.0", headers)
assert not connection_close(b"HTTP/1.1", headers)
def flow():
request = Request(host=b'localhost',
path=b'/test/',
http_version=b'1.1',
port=1234,
method=b'',
scheme=b'',
headers=Headers([(b"Host", b"example.com")]),
content=None,
timestamp_start=111.1,
timestamp_end=111.2,
authority=b'',
trailers='')
flow = HTTPFlow(client_conn=MagicMock(), server_conn=MagicMock())
flow.request = request
return flow
def response(flow: http.HTTPFlow):
if flow.response.trailers:
print("HTTP Trailers detected! Response contains:",
flow.response.trailers)
if flow.request.path == "/inject_trailers":
if flow.request.is_http10:
return
elif flow.request.is_http11:
if not flow.response.content:
return
flow.response.headers["transfer-encoding"] = "chunked"
flow.response.headers["trailer"] = "x-my-injected-trailer-header"
flow.response.trailers = Headers([(b"x-my-injected-trailer-header",
b"foobar")])
print("Injected a new response trailer...",
flow.response.headers["trailer"])
