def test_echo_trailer(): sio = io.StringIO() sio_err = io.StringIO() d = dumper.Dumper(sio, sio_err) with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3) f = tflow.tflow(client_conn=True, server_conn=True, resp=True) f.request.headers["content-type"] = "text/html" f.request.headers["transfer-encoding"] = "chunked" f.request.headers["trailer"] = "my-little-request-trailer" f.request.content = b"some request content\n" * 100 f.request.trailers = Headers([(b"my-little-request-trailer", b"foobar-request-trailer")]) f.response.headers["transfer-encoding"] = "chunked" f.response.headers["trailer"] = "my-little-response-trailer" f.response.content = b"some response content\n" * 100 f.response.trailers = Headers([(b"my-little-response-trailer", b"foobar-response-trailer")]) d.echo_flow(f) t = sio.getvalue() assert "content-type" in t assert "cut off" in t assert "some request content" in t assert "foobar-request-trailer" in t assert "some response content" in t assert "foobar-response-trailer" in t
def test_insert(self): headers = Headers(Accept="text/plain") headers.insert(0, b"Host", "example.com") assert headers.fields == ( (b'Host', b'example.com'), (b'Accept', b'text/plain') )
def test_get_header_tokens(): headers = Headers() assert get_header_tokens(headers, "foo") == [] headers["foo"] = "bar" assert get_header_tokens(headers, "foo") == ["bar"] headers["foo"] = "bar, voing" assert get_header_tokens(headers, "foo") == ["bar", "voing"] headers.set_all("foo", ["bar, voing", "oink"]) assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"]
def test_bytes(self): headers = Headers(Host="example.com") assert bytes(headers) == b"Host: example.com\r\n" headers = Headers([(b"Host", b"example.com"), (b"Accept", b"text/plain")]) assert bytes(headers) == b"Host: example.com\r\nAccept: text/plain\r\n" headers = Headers() assert bytes(headers) == b""
def test_validate_headers(): # both content-length and chunked (possible request smuggling) with pytest.raises( ValueError, match= "Received both a Transfer-Encoding and a Content-Length header"): validate_headers( Headers(transfer_encoding="chunked", content_length="42"), ) with pytest.raises(ValueError, match="Received an invalid header name"): validate_headers(Headers([(b"content-length ", b"42")]), )
def test_items(self): headers = Headers([ (b"Set-Cookie", b"foo"), (b"Set-Cookie", b"bar"), (b'Accept', b'text/plain'), ]) assert list(headers.items()) == [('Set-Cookie', 'foo, bar'), ('Accept', 'text/plain')] assert list(headers.items(multi=True)) == [('Set-Cookie', 'foo'), ('Set-Cookie', 'bar'), ('Accept', 'text/plain')]
def test_make(self): r = Request.make("GET", "https://example.com/") assert r.method == "GET" assert r.scheme == "https" assert r.host == "example.com" assert r.port == 443 assert r.path == "/" r = Request.make("GET", "https://example.com/", "content", {"Foo": "bar"}) assert r.content == b"content" assert r.headers["content-length"] == "7" assert r.headers["Foo"] == "bar" Request.make("GET", "https://example.com/", content=b"content") with pytest.raises(TypeError): Request.make("GET", "https://example.com/", content=42) r = Request.make("GET", "https://example.com/", headers=[(b"foo", b"bar")]) assert r.headers["foo"] == "bar" r = Request.make("GET", "https://example.com/", headers=({"foo": "baz"})) assert r.headers["foo"] == "baz" r = Request.make("GET", "https://example.com/", headers=Headers(foo="qux")) assert r.headers["foo"] == "qux" with pytest.raises(TypeError): Request.make("GET", "https://example.com/", headers=42)
def _read_headers(lines: Iterable[bytes]) -> Headers: """ Read a set of headers. Stop once a blank line is reached. Returns: A headers object Raises: exceptions.HttpSyntaxException """ ret: List[Tuple[bytes, bytes]] = [] for line in lines: if line[0] in b" \t": if not ret: raise ValueError("Invalid headers") # continued header ret[-1] = (ret[-1][0], ret[-1][1] + b'\r\n ' + line.strip()) else: try: name, value = line.split(b":", 1) value = value.strip() if not name: raise ValueError() ret.append((name, value)) except ValueError: raise ValueError(f"Invalid header line: {line!r}") return Headers(ret)
def test_serializable(self): data1 = tresp(timestamp_start=42, timestamp_end=42).data data1.trailers = Headers() data2 = tresp().data.from_state( data1.get_state()) # ResponseData.from_state() assert data1 == data2
def test_content_length_not_added_for_response_with_transfer_encoding( self): headers = Headers(((b"transfer-encoding", b"chunked"), )) resp = tresp(headers=headers) resp.content = b"bar" assert "content-length" not in resp.headers
def test_get_cookies_withequalsign(self): request = treq() request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue") result = request.cookies assert len(result) == 2 assert result['cookiename'] == 'coo=kievalue' assert result['othercookiename'] == 'othercookievalue'
def test_make(self): r = Response.make() assert r.status_code == 200 assert r.content == b"" r = Response.make(418, "teatime") assert r.status_code == 418 assert r.content == b"teatime" assert r.headers["content-length"] == "7" Response.make(content=b"foo") Response.make(content="foo") with pytest.raises(TypeError): Response.make(content=42) r = Response.make(headers=[(b"foo", b"bar")]) assert r.headers["foo"] == "bar" r = Response.make(headers=({"foo": "baz"})) assert r.headers["foo"] == "baz" r = Response.make(headers=Headers(foo="qux")) assert r.headers["foo"] == "qux" with pytest.raises(TypeError): Response.make(headers=42)
def test_get_cookies_simple(self): resp = tresp() resp.headers = Headers(set_cookie="cookiename=cookievalue") result = resp.cookies assert len(result) == 1 assert "cookiename" in result assert result["cookiename"] == ("cookievalue", CookieAttrs())
def _2host(self): return Headers( ( (b"Host", b"example.com"), (b"host", b"example.org") ) )
def test_set(self): headers = Headers() headers["foo"] = "1" headers[b"bar"] = b"2" headers["baz"] = b"3" with pytest.raises(TypeError): headers["foobar"] = 42 assert len(headers) == 3
def run(self, flow: http.HTTPFlow, hdrs: Headers) -> None: # unset all specified headers for spec in self.replacements: if spec.matches(flow): hdrs.pop(spec.subject, None) # set all specified headers if the replacement string is not empty for spec in self.replacements: if spec.matches(flow): try: replacement = spec.read_replacement() except OSError as e: ctx.log.warn(f"Could not read replacement file: {e}") continue else: if replacement: hdrs.add(spec.subject, replacement)
def test_get_cookies_no_value(self): resp = tresp() resp.headers = Headers(set_cookie="cookiename=; Expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/") result = resp.cookies assert len(result) == 1 assert "cookiename" in result assert result["cookiename"][0] == "" assert len(result["cookiename"][1]) == 2
def __convert_headers_for_mitm(self, headers): headers_obj = json.loads(headers) headers_list = [] for key, value in headers_obj.items(): header = (key.encode(), value.encode()) headers_list.append(header) return Headers(headers_list)
def test_set_cookies(self): request = treq() request.headers = Headers(cookie="cookiename=cookievalue") result = request.cookies result["cookiename"] = "foo" assert request.cookies["cookiename"] == "foo" request.cookies = [["one", "uno"], ["two", "due"]] assert request.cookies["one"] == "uno" assert request.cookies["two"] == "due"
def test_encode(): data = [(b"file", b"shell.jpg"), (b"file_size", b"1000")] headers = Headers( content_type='multipart/form-data; boundary=127824672498') content = multipart.encode(headers, data) assert b'Content-Disposition: form-data; name="file"' in content assert b'Content-Type: text/plain; charset=utf-8\r\n\r\nshell.jpg\r\n\r\n--127824672498\r\n' in content assert b'1000\r\n\r\n--127824672498--\r\n' assert len(content) == 252 with pytest.raises(ValueError, match=r"boundary found in encoded string"): multipart.encode(headers, [(b"key", b"--127824672498")]) boundary = 'boundary茅莽' headers = Headers(content_type='multipart/form-data; boundary=' + boundary) result = multipart.encode(headers, data) assert result == b''
def test_get_cookies_twocookies(self): resp = tresp() resp.headers = Headers([[b"Set-Cookie", b"cookiename=cookievalue"], [b"Set-Cookie", b"othercookie=othervalue"]]) result = resp.cookies assert len(result) == 2 assert "cookiename" in result assert result["cookiename"] == ("cookievalue", CookieAttrs()) assert "othercookie" in result assert result["othercookie"] == ("othervalue", CookieAttrs())
def test_generate_tflow_js(tdata): tf_http = tflow.tflow(resp=True, err=True, ws=True) tf_http.id = "d91165be-ca1f-4612-88a9-c0f8696f3e29" tf_http.client_conn.id = "4a18d1a0-50a1-48dd-9aa6-d45d74282939" tf_http.server_conn.id = "f087e7b2-6d0a-41a8-a8f0-e1a4761395f8" tf_http.server_conn.certificate_list = [ certs.Cert.from_pem( Path( tdata.path( "mitmproxy/net/data/verificationcerts/self-signed.pem")). read_bytes()) ] tf_http.request.trailers = Headers(trailer="qvalue") tf_http.response.trailers = Headers(trailer="qvalue") tf_http.comment = "I'm a comment!" tf_tcp = tflow.ttcpflow(err=True) tf_tcp.id = "2ea7012b-21b5-4f8f-98cd-d49819954001" tf_tcp.client_conn.id = "8be32b99-a0b3-446e-93bc-b29982fe1322" tf_tcp.server_conn.id = "e33bb2cd-c07e-4214-9a8e-3a8f85f25200" # language=TypeScript content = ( "/** Auto-generated by test_app.py:test_generate_tflow_js */\n" "import {HTTPFlow, TCPFlow} from '../../flow';\n" "export function THTTPFlow(): Required<HTTPFlow> {\n" " return %s\n" "}\n" "export function TTCPFlow(): Required<TCPFlow> {\n" " return %s\n" "}" % ( textwrap.indent( json.dumps(app.flow_to_json(tf_http), indent=4, sort_keys=True), " "), textwrap.indent( json.dumps(app.flow_to_json(tf_tcp), indent=4, sort_keys=True), " "), )) content = content.replace(": null", ": undefined") (Path(__file__).parent / "../../../../web/src/js/__tests__/ducks/_tflow.ts").write_bytes( content.encode())
def test_assemble_body(): c = list(assemble_body(Headers(), [b"body"], Headers())) assert c == [b"body"] c = list( assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a", b""], Headers())) assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"] c = list( assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers())) assert c == [b"a\r\n123456789a\r\n", b"0\r\n\r\n"] c = list( assemble_body(Headers(transfer_encoding="chunked"), [b"123456789a"], Headers(trailer="trailer"))) assert c == [b"a\r\n123456789a\r\n", b"0\r\ntrailer: trailer\r\n\r\n"] with pytest.raises(ValueError): list(assemble_body(Headers(), [b"body"], Headers(trailer="trailer")))
def test_init(self): headers = Headers() assert len(headers) == 0 headers = Headers([(b"Host", b"example.com")]) assert len(headers) == 1 assert headers["Host"] == "example.com" headers = Headers(Host="example.com") assert len(headers) == 1 assert headers["Host"] == "example.com" headers = Headers( [(b"Host", b"invalid")], Host="example.com" ) assert len(headers) == 1 assert headers["Host"] == "example.com" headers = Headers( [(b"Host", b"invalid"), (b"Accept", b"text/plain")], Host="example.com" ) assert len(headers) == 2 assert headers["Host"] == "example.com" assert headers["Accept"] == "text/plain" with pytest.raises(TypeError): Headers([(b"Host", "not-bytes")])
def test_modify_form(self, tdata): with taddons.context() as tctx: sc = tctx.script(tdata.path("../examples/addons/http-modify-form.py")) form_header = Headers(content_type="application/x-www-form-urlencoded") f = tflow.tflow(req=tutils.treq(headers=form_header)) sc.request(f) assert f.request.urlencoded_form["mitmproxy"] == "rocks" f.request.headers["content-type"] = "" sc.request(f) assert list(f.request.urlencoded_form.items()) == [("foo", "bar")]
def test_get_cookies_with_parameters(self): resp = tresp() cookie = "cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly" resp.headers = Headers(set_cookie=cookie) result = resp.cookies assert len(result) == 1 assert "cookiename" in result assert result["cookiename"][0] == "cookievalue" attrs = result["cookiename"][1] assert len(attrs) == 4 assert attrs["domain"] == "example.com" assert attrs["expires"] == "Wed Oct 21 16:29:41 2015" assert attrs["path"] == "/" assert attrs["httponly"] == ""
def run(self, flow: http.HTTPFlow, hdrs: Headers) -> None: matches = [] # first check all the filters against the original, unmodified flow for spec in self.replacements: matches.append(spec.matches(flow)) # unset all specified headers for i, spec in enumerate(self.replacements): if matches[i]: hdrs.pop(spec.subject, None) # set all specified headers if the replacement string is not empty for i, spec in enumerate(self.replacements): if matches[i]: try: replacement = spec.read_replacement() except OSError as e: ctx.log.warn(f"Could not read replacement file: {e}") continue else: if replacement: hdrs.add(spec.subject, replacement)
def test_connection_close(): headers = Headers() assert connection_close(b"HTTP/1.0", headers) assert not connection_close(b"HTTP/1.1", headers) assert not connection_close(b"HTTP/2.0", headers) headers["connection"] = "keep-alive" assert not connection_close(b"HTTP/1.1", headers) headers["connection"] = "close" assert connection_close(b"HTTP/1.1", headers) headers["connection"] = "foobar" assert connection_close(b"HTTP/1.0", headers) assert not connection_close(b"HTTP/1.1", headers)
def flow(): request = Request(host=b'localhost', path=b'/test/', http_version=b'1.1', port=1234, method=b'', scheme=b'', headers=Headers([(b"Host", b"example.com")]), content=None, timestamp_start=111.1, timestamp_end=111.2, authority=b'', trailers='') flow = HTTPFlow(client_conn=MagicMock(), server_conn=MagicMock()) flow.request = request return flow
def response(flow: http.HTTPFlow): if flow.response.trailers: print("HTTP Trailers detected! Response contains:", flow.response.trailers) if flow.request.path == "/inject_trailers": if flow.request.is_http10: return elif flow.request.is_http11: if not flow.response.content: return flow.response.headers["transfer-encoding"] = "chunked" flow.response.headers["trailer"] = "x-my-injected-trailer-header" flow.response.trailers = Headers([(b"x-my-injected-trailer-header", b"foobar")]) print("Injected a new response trailer...", flow.response.headers["trailer"])