async def test_storage_bodies(self): # Need to test for configure # Need to test for set_order s = self.start_session(fp=0.5) f = self.tft() f2 = self.tft(start=1) f.request.content = b"A" * 1001 s.request(f) s.request(f2) await asyncio.sleep(1.0) content = s.db_store.con.execute( "SELECT type_id, content FROM body WHERE body.flow_id == (?);", [f.id] ).fetchall()[0] assert content == (1, b"A" * 1001) assert s.db_store.body_ledger == {f.id} f.response = http.HTTPResponse.wrap(tutils.tresp(content=b"A" * 1001)) f2.response = http.HTTPResponse.wrap(tutils.tresp(content=b"A" * 1001)) # Content length is wrong for some reason -- quick fix f.response.headers['content-length'] = b"1001" f2.response.headers['content-length'] = b"1001" s.response(f) s.response(f2) await asyncio.sleep(1.0) rows = s.db_store.con.execute( "SELECT type_id, content FROM body WHERE body.flow_id == (?);", [f.id] ).fetchall() assert len(rows) == 1 rows = s.db_store.con.execute( "SELECT type_id, content FROM body WHERE body.flow_id == (?);", [f2.id] ).fetchall() assert len(rows) == 1 assert s.db_store.body_ledger == {f.id} assert all([lf.__dict__ == rf.__dict__ for lf, rf in list(zip(s.load_view(), [f, f2]))])
def test_eq(self): data = tutils.tresp(timestamp_start=42, timestamp_end=42).data same = tutils.tresp(timestamp_start=42, timestamp_end=42).data assert data == same other = tutils.tresp(content=b"foo").data assert data != other assert data != 0
def test_eq_ne(self): resp = tutils.tresp(timestamp_start=42, timestamp_end=42) same = tutils.tresp(timestamp_start=42, timestamp_end=42) assert resp == same other = tutils.tresp(timestamp_start=0, timestamp_end=0) assert resp != other assert resp != 0
def test_assemble_response(): assert assemble_response(tresp()) == ( b"HTTP/1.1 200 OK\r\n" b"header-response: svalue\r\n" b"content-length: 7\r\n" b"\r\n" b"message" ) with raises(exceptions.HttpException): assemble_response(tresp(content=None))
def test_expected_http_body_size(): # Expect: 100-continue assert expected_http_body_size( treq(headers=Headers(expect="100-continue", content_length="42")) ) == 0 # http://tools.ietf.org/html/rfc7230#section-3.3 assert expected_http_body_size( treq(method=b"HEAD"), tresp(headers=Headers(content_length="42")) ) == 0 assert expected_http_body_size( treq(method=b"CONNECT"), tresp() ) == 0 for code in (100, 204, 304): assert expected_http_body_size( treq(), tresp(status_code=code) ) == 0 # chunked assert expected_http_body_size( treq(headers=Headers(transfer_encoding="chunked")), ) is None # explicit length for val in (b"foo", b"-7"): with pytest.raises(exceptions.HttpSyntaxException): expected_http_body_size( treq(headers=Headers(content_length=val)) ) assert expected_http_body_size( treq(headers=Headers(content_length="42")) ) == 42 # more than 1 content-length headers with same value assert expected_http_body_size( treq(headers=Headers([(b'content-length', b'42'), (b'content-length', b'42')])) ) == 42 # more than 1 content-length headers with conflicting value with pytest.raises(exceptions.HttpSyntaxException): expected_http_body_size( treq(headers=Headers([(b'content-length', b'42'), (b'content-length', b'45')])) ) # no length assert expected_http_body_size( treq(headers=Headers()) ) == 0 assert expected_http_body_size( treq(headers=Headers()), tresp(headers=Headers()) ) == -1
def test_init(self): with pytest.raises(ValueError): tresp(headers="foobar") with pytest.raises(UnicodeEncodeError): tresp(http_version="föö/bä.r") with pytest.raises(UnicodeEncodeError): tresp(reason="fööbär") with pytest.raises(ValueError): tresp(content="foobar") assert isinstance(tresp(headers=()).headers, Headers)
def test_urldict_get(self, tmpdir): tmpfile = tmpdir.join("tmpfile") with open(tmpfile, "w") as tfile: tfile.write(input_file_content) with open(tmpfile, "r") as tfile: urldict = URLDict.load(tfile) f = tflow.tflow(resp=tutils.tresp()) f.request.url = url selection = urldict[f] assert "body" in selection[0] assert new_content_body in selection[0]["body"] assert "title" in selection[0] assert new_content_title in selection[0]["title"] selection_get = urldict.get(f) assert "body" in selection_get[0] assert new_content_body in selection_get[0]["body"] assert "title" in selection_get[0] assert new_content_title in selection_get[0]["title"] try: urldict["body"] except KeyError: assert True else: assert False assert urldict.get("body", default="default") == "default"
def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): """ @type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection @type server_conn: bool | None | mitmproxy.proxy.connection.ServerConnection @type req: bool | None | mitmproxy.proxy.protocol.http.HTTPRequest @type resp: bool | None | mitmproxy.proxy.protocol.http.HTTPResponse @type err: bool | None | mitmproxy.proxy.protocol.primitives.Error @return: mitmproxy.proxy.protocol.http.HTTPFlow """ if client_conn is True: client_conn = tclient_conn() if server_conn is True: server_conn = tserver_conn() if req is True: req = tutils.treq() if resp is True: resp = tutils.tresp() if err is True: err = terr() f = http.HTTPFlow(client_conn, server_conn) f.request = req f.response = resp f.error = err f.reply = controller.DummyReply() return f
def test_get_cookies_simple(self): resp = tresp() resp.headers = Headers(set_cookie="cookiename=cookievalue") result = resp.cookies assert len(result) == 1 assert "cookiename" in result assert result["cookiename"] == ("cookievalue", CookieAttrs())
def test_cannot_encode(self): r = tutils.tresp() r.content = None assert "content-type" not in r.headers assert r.raw_content is None r.headers["content-type"] = "text/html; charset=latin1; foo=bar" r.text = u"☃" assert r.headers["content-type"] == "text/html; charset=utf-8; foo=bar" assert r.raw_content == b'\xe2\x98\x83' r.headers["content-type"] = "gibberish" r.text = u"☃" assert r.headers["content-type"] == "text/plain; charset=utf-8" assert r.raw_content == b'\xe2\x98\x83' del r.headers["content-type"] r.text = u"☃" assert r.headers["content-type"] == "text/plain; charset=utf-8" assert r.raw_content == b'\xe2\x98\x83' r.headers["content-type"] = "text/html; charset=latin1" r.text = u'\udcff' assert r.headers["content-type"] == "text/html; charset=utf-8" assert r.raw_content == b"\xFF"
def test_unknown_ce(self): r = tutils.tresp() r.headers["content-type"] = "text/html; charset=wtf" r.raw_content = b"foo" with pytest.raises(ValueError): assert r.text == u"foo" assert r.get_text(strict=False) == u"foo"
def test_none(self): r = tutils.tresp(content=None) assert r.text is None r.text = u"foo" assert r.text is not None r.text = None assert r.text is None
def test_update_content_length_header(self): r = tutils.tresp() assert int(r.headers["content-length"]) == 7 r.encode("gzip") assert int(r.headers["content-length"]) == 27 r.decode() assert int(r.headers["content-length"]) == 7
def test_none(self): r = tresp(content=None) assert r.content is None r.content = b"foo" assert r.content is not None r.content = None assert r.content is None
def test_response(self, monkeypatch, logger): logger.args = [] monkeypatch.setattr("mitmproxy.ctx.log", logger) monkeypatch.setattr(requests, 'get', self.mocked_requests_invuln) mocked_flow = tflow.tflow(req=tutils.treq(path=b"index.html?q=1"), resp=tutils.tresp(content=b'<html></html>')) xss.response(mocked_flow) assert logger.args == []
def test_set_cookies(self): resp = tresp() resp.cookies["foo"] = ("bar", {}) assert len(resp.cookies) == 1 assert resp.cookies["foo"] == ("bar", CookieAttrs()) resp.cookies = [["one", ("uno", CookieAttrs())], ["two", ("due", CookieAttrs())]] assert list(resp.cookies.keys()) == ["one", "two"]
def test_inject_not404(self): sitemap_injection = SitemapInjection() f = tflow.tflow(resp=tutils.tresp()) with mock.patch.object(logger, 'warning') as mock_warning: sitemap_injection.inject(index, f) assert mock_warning.called
def test_inject_404(self): robots_injection = RobotsInjection() f = tflow.tflow(resp=tutils.tresp()) f.response.status_code = 404 assert "Allow: /test" not in str(f.response.content) robots_injection.inject(index, f) assert "Allow: /test" in str(f.response.content)
def test_inject_404(self): html_injection = HTMLInjection() f = tflow.tflow(resp=tutils.tresp()) f.response.status_code = 404 assert "example.com" not in str(f.response.content) html_injection.inject(index, f) assert "example.com" in str(f.response.content)
def test_inject_insert_body(self): html_injection = HTMLInjection(insert=True) f = tflow.tflow(resp=tutils.tresp()) f.response.text = "<body></body>" assert "example.com" not in str(f.response.content) html_injection.inject(index, f) assert "example.com" in str(f.response.content)
def test_response(self, get_request_invuln, logger): mocked_flow = tflow.tflow( req=tutils.treq(path=b"index.html?q=1"), resp=tutils.tresp(content=b'<html></html>') ) xss.response(mocked_flow) assert logger.args == []
async def test_storage_flush_with_specials(self): s = self.start_session(fp=0.5) f = self.tft() s.request(f) await asyncio.sleep(1) assert len(s._hot_store) == 0 f.response = http.HTTPResponse.wrap(tutils.tresp()) s.response(f) assert len(s._hot_store) == 1 assert s.load_storage() == [f] await asyncio.sleep(1) assert all([lflow.__dict__ == flow.__dict__ for lflow, flow in list(zip(s.load_storage(), [f]))]) f.server_conn.via = tflow.tserver_conn() s.request(f) await asyncio.sleep(0.6) assert len(s._hot_store) == 0 assert all([lflow.__dict__ == flow.__dict__ for lflow, flow in list(zip(s.load_storage(), [f]))]) flows = [self.tft() for _ in range(500)] s.update(flows) await asyncio.sleep(0.6) assert s._flush_period == s._FP_DEFAULT * s._FP_DECREMENT await asyncio.sleep(3) assert s._flush_period == s._FP_DEFAULT
def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): """ @type client_conn: bool | None | mitmproxy.proxy.connection.ClientConnection @type server_conn: bool | None | mitmproxy.proxy.connection.ServerConnection @type req: bool | None | mitmproxy.proxy.protocol.http.HTTPRequest @type resp: bool | None | mitmproxy.proxy.protocol.http.HTTPResponse @type err: bool | None | mitmproxy.proxy.protocol.primitives.Error @return: mitmproxy.proxy.protocol.http.HTTPFlow """ if client_conn is True: client_conn = tclient_conn() if server_conn is True: server_conn = tserver_conn() if req is True: req = tutils.treq() if resp is True: resp = tutils.tresp() if err is True: err = terr() if req: req = http.HTTPRequest.wrap(req) if resp: resp = http.HTTPResponse.wrap(resp) f = http.HTTPFlow(client_conn, server_conn) f.request = req f.response = resp f.error = err f.reply = controller.DummyReply() return f
def tflow( *, client_conn: Optional[connection.Client] = None, server_conn: Optional[connection.Server] = None, req: Optional[http.Request] = None, resp: Union[bool, http.Response] = False, err: Union[bool, flow.Error] = False, ws: Union[bool, websocket.WebSocketData] = False, ) -> http.HTTPFlow: """Create a flow for testing.""" if client_conn is None: client_conn = tclient_conn() if server_conn is None: server_conn = tserver_conn() if req is None: req = treq() if resp is True: resp = tresp() if err is True: err = terr() if ws is True: ws = twebsocket() assert resp is False or isinstance(resp, http.Response) assert err is False or isinstance(err, flow.Error) assert ws is False or isinstance(ws, websocket.WebSocketData) f = http.HTTPFlow(client_conn, server_conn) f.request = req f.response = resp or None f.error = err or None f.websocket = ws or None f.reply = controller.DummyReply() return f
def test_expected_http_body_size(): # Expect: 100-continue assert expected_http_body_size( treq(headers=Headers(expect="100-continue", content_length="42")), expect_continue_as_0=True) == 0 # Expect: 100-continue assert expected_http_body_size( treq(headers=Headers(expect="100-continue", content_length="42")), expect_continue_as_0=False) == 42 # http://tools.ietf.org/html/rfc7230#section-3.3 assert expected_http_body_size( treq(method=b"HEAD"), tresp(headers=Headers(content_length="42"))) == 0 assert expected_http_body_size( treq(method=b"CONNECT"), None, ) == 0 assert expected_http_body_size(treq(method=b"CONNECT"), tresp()) == 0 for code in (100, 204, 304): assert expected_http_body_size(treq(), tresp(status_code=code)) == 0 # chunked assert expected_http_body_size( treq(headers=Headers(transfer_encoding="chunked")), ) is None # explicit length for val in (b"foo", b"-7"): with pytest.raises(ValueError): expected_http_body_size(treq(headers=Headers(content_length=val))) assert expected_http_body_size( treq(headers=Headers(content_length="42"))) == 42 # more than 1 content-length headers with same value assert expected_http_body_size( treq(headers=Headers([(b'content-length', b'42'), (b'content-length', b'42')]))) == 42 # more than 1 content-length headers with conflicting value with pytest.raises(ValueError): expected_http_body_size( treq(headers=Headers([(b'content-length', b'42'), (b'content-length', b'45')]))) # no length assert expected_http_body_size(treq(headers=Headers())) == 0 assert expected_http_body_size(treq(headers=Headers()), tresp(headers=Headers())) == -1
def test_unknown_ce(self): r = tutils.tresp() r.headers["content-encoding"] = "zopfli" r.raw_content = b"foo" with tutils.raises(ValueError): assert r.content assert r.headers["content-encoding"] assert r.get_content(strict=False) == b"foo"
def test_get_cookies_no_value(self): resp = tresp() resp.headers = Headers(set_cookie="cookiename=; Expires=Thu, 01-Jan-1970 00:00:01 GMT; path=/") result = resp.cookies assert len(result) == 1 assert "cookiename" in result assert result["cookiename"][0] == "" assert len(result["cookiename"][1]) == 2
def test_cannot_decode(self): r = tutils.tresp() r.headers["content-type"] = "text/html; charset=utf8" r.raw_content = b"\xFF" with pytest.raises(ValueError): assert r.text assert r.get_text(strict=False) == '\udcff'
def test_utf8_as_ce(self): r = tutils.tresp() r.headers["content-encoding"] = "utf8" r.raw_content = b"foo" with pytest.raises(ValueError): assert r.content assert r.headers["content-encoding"] assert r.get_content(strict=False) == b"foo"
def test_unknown_ce(self): r = tresp() r.headers["content-encoding"] = "zopfli" r.raw_content = b"foo" with pytest.raises(ValueError): assert r.content assert r.headers["content-encoding"] assert r.get_content(strict=False) == b"foo"
def test_add(self, tmpdir): tmpfile = tmpdir.join("tmpfile") writer = JSONUrlIndexWriter(filename=tmpfile) f = tflow.tflow(resp=tutils.tresp()) url = f"{f.request.scheme}://{f.request.host}:{f.request.port}" writer.add_url(f) assert url in writer.host_urls assert f.request.path in writer.host_urls[url]
def test_response(self, tmpdir): tmpfile = tmpdir.join("tmpfile") url_index = UrlIndexAddon(tmpfile) f = tflow.tflow(resp=tutils.tresp()) with mock.patch( 'examples.complex.webscanner_helper.urlindex.JSONUrlIndexWriter.add_url' ) as mock_add_url: url_index.response(f) mock_add_url.assert_called()
def flow(self, resp_content=b'message'): times = dict( timestamp_start=746203272, timestamp_end=746203272, ) # Create a dummy flow for testing return tflow.tflow(req=tutils.treq(method=b'GET', **times), resp=tutils.tresp(content=resp_content, **times))
def test_request(self, selenium_addon): f = tflow.tflow(resp=tutils.tresp()) f.request.url = "http://example.com/login.php" selenium_addon.set_cookies = False assert not selenium_addon.set_cookies with mock.patch.object(logger, 'debug') as mock_debug: selenium_addon.request(f) mock_debug.assert_called() assert selenium_addon.set_cookies
def test_response(self, selenium_addon): f = tflow.tflow(resp=tutils.tresp()) f.request.url = "http://example.com/login.php" selenium_addon.set_cookies = False with mock.patch( 'examples.complex.webscanner_helper.proxyauth_selenium.SeleniumAddon.login', return_value=[]) as mock_login: selenium_addon.response(f) mock_login.assert_called()
def test_iframe_injector(self): with tutils.raises(ScriptError): tscript("iframe_injector.py") m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe") f = tflow.tflow(resp=tutils.tresp(content=b"<html>mitmproxy</html>")) m.response(f) content = f.response.content assert b'iframe' in content and b'evil_iframe' in content
def test_iframe_injector(self): with tutils.raises(ScriptError): tscript("simple/modify_body_inject_iframe.py") m, sc = tscript("simple/modify_body_inject_iframe.py", "http://example.org/evil_iframe") f = tflow.tflow(resp=tutils.tresp(content=b"<html><body>mitmproxy</body></html>")) m.response(f) content = f.response.content assert b'iframe' in content and b'evil_iframe' in content
def test_replace(self): r = tutils.tresp() r.content = b"foofootoo" r.replace(b"foo", "gg") assert r.content == b"ggggtoo" r.content = b"foofootoo" r.replace(b"foo", "gg", count=1) assert r.content == b"ggfootoo"
def test_inject_404(self): sitemap_injection = SitemapInjection() f = tflow.tflow(resp=tutils.tresp()) f.response.status_code = 404 assert "<url><loc>http://example.com:80/</loc></url>" not in str( f.response.content) sitemap_injection.inject(index, f) assert "<url><loc>http://example.com:80/</loc></url>" in str( f.response.content)
def cycle(self, master, content): f = tflow.tflow(req=tutils.treq(content=content)) master.clientconnect(f.client_conn) master.serverconnect(f.server_conn) master.request(f) if not f.error: f.response = http.HTTPResponse.wrap(tutils.tresp(content=content)) master.response(f) master.clientdisconnect(f) return f
def test_modify(self): r = tutils.tresp() r.text = u"ü" assert r.raw_content == b"\xfc" r.headers["content-type"] = "text/html; charset=utf8" r.text = u"ü" assert r.raw_content == b"\xc3\xbc" assert r.headers["content-length"] == "2"
def test_simple(self): r = tutils.tresp() assert r.raw_content == b"message" assert "content-encoding" not in r.headers r.encode("gzip") assert r.headers["content-encoding"] assert r.raw_content != b"message" assert r.content == b"message" assert r.raw_content != b"message"
def test_content_length_update(self): resp = tutils.tresp() resp.content = b"foo" assert resp.data.content == b"foo" assert resp.headers["content-length"] == "3" resp.content = b"" assert resp.data.content == b"" assert resp.headers["content-length"] == "0" resp.raw_content = b"bar" assert resp.data.content == b"bar" assert resp.headers["content-length"] == "0"
def flow(self, resp_content=b'message'): times = dict( timestamp_start=746203272, timestamp_end=746203272, ) # Create a dummy flow for testing return tflow.tflow( req=tutils.treq(method=b'GET', **times), resp=tutils.tresp(content=resp_content, **times) )
def test_modify(self): r = tutils.tresp() assert "content-encoding" not in r.headers r.encode("gzip") r.content = b"foo" assert r.raw_content != b"foo" r.decode() assert r.raw_content == b"foo" with pytest.raises(TypeError): r.content = u"foo"
def cycle(self, master, content): f = tflow.tflow(req=tutils.treq(content=content)) master.clientconnect(f.client_conn) master.serverconnect(f.server_conn) master.request(f) if not f.error: f.response = http.HTTPResponse.wrap( tutils.tresp(content=content) ) master.response(f) master.clientdisconnect(f) return f
def test_get_cookies_twocookies(self): resp = tresp() resp.headers = Headers([ [b"Set-Cookie", b"cookiename=cookievalue"], [b"Set-Cookie", b"othercookie=othervalue"] ]) result = resp.cookies assert len(result) == 2 assert "cookiename" in result assert result["cookiename"] == ("cookievalue", CookieAttrs()) assert "othercookie" in result assert result["othercookie"] == ("othervalue", CookieAttrs())
def test_intercept(self): """regression test for https://github.com/mitmproxy/mitmproxy/issues/1605""" m = self.mkmaster(intercept="~b bar") f = tflow.tflow(req=tutils.treq(content=b"foo")) m.addons.handle_lifecycle("request", f) assert not m.view[0].intercepted f = tflow.tflow(req=tutils.treq(content=b"bar")) m.addons.handle_lifecycle("request", f) assert m.view[1].intercepted f = tflow.tflow(resp=tutils.tresp(content=b"bar")) m.addons.handle_lifecycle("request", f) assert m.view[2].intercepted