def test_decodeencode(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" assert r.decode() assert not r.headers["content-encoding"] assert r.content == "falafel" r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("identity") assert r.headers["content-encoding"] == ["identity"] assert r.content == "falafel" r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("gzip") assert r.headers["content-encoding"] == ["gzip"] assert r.content != "falafel" assert r.decode() assert not r.headers["content-encoding"] assert r.content == "falafel" r.headers["content-encoding"] = ["gzip"] assert not r.decode() assert r.content == "falafel"
def test_flow(self): """ normal flow: connect -> request -> response """ c = flow.State() f = tutils.tflow() c.add_flow(f) assert f assert c.flow_count() == 1 assert c.active_flow_count() == 1 newf = tutils.tflow() assert c.add_flow(newf) assert c.active_flow_count() == 2 f.response = HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(f) assert c.flow_count() == 2 assert c.active_flow_count() == 1 _ = HTTPResponse.wrap(netlib.tutils.tresp()) assert not c.update_flow(None) assert c.active_flow_count() == 1 newf.response = HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(newf) assert c.active_flow_count() == 0
def test_decodeencode(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = "identity" r.content = "falafel" assert r.decode() assert "content-encoding" not in r.headers assert r.content == "falafel" r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = "identity" r.content = "falafel" r.encode("identity") assert r.headers["content-encoding"] == "identity" assert r.content == "falafel" r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = "identity" r.content = "falafel" r.encode("gzip") assert r.headers["content-encoding"] == "gzip" assert r.content != "falafel" assert r.decode() assert "content-encoding" not in r.headers assert r.content == "falafel" r.headers["content-encoding"] = "gzip" assert not r.decode() assert r.content == "falafel"
def block_request(flow): """ Modifies flow and blocks request for suspected proxy usage """ resp = HTTPResponse("HTTP/1.1", 200, "OK", Headers(Context_Type="text/html"), block_html) flow.reply(resp)
def test_replace(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["Foo"] = "fOo" r.content = "afoob" assert r.replace("foo(?i)", "boo") == 3 assert not "foo" in r.content assert r.headers["boo"] == "boo"
def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): """ @type client_conn: bool | None | libmproxy.proxy.connection.ClientConnection @type server_conn: bool | None | libmproxy.proxy.connection.ServerConnection @type req: bool | None | libmproxy.protocol.http.HTTPRequest @type resp: bool | None | libmproxy.protocol.http.HTTPResponse @type err: bool | None | libmproxy.protocol.primitives.Error @return: bool | None | libmproxy.protocol.http.HTTPFlow """ if client_conn is True: client_conn = tclient_conn() if server_conn is True: server_conn = tserver_conn() if req is True: req = netlib.tutils.treq() if resp is True: resp = netlib.tutils.tresp() if err is True: err = terr() if req: req = HTTPRequest.wrap(req) if resp: resp = HTTPResponse.wrap(resp) f = HTTPFlow(client_conn, server_conn) f.request = req f.response = resp f.error = err f.reply = controller.DummyReply() return f
def test_set_limit(self): c = flow.State() f = tutils.tflow() assert len(c.view) == 0 c.add_flow(f) assert len(c.view) == 1 c.set_limit("~s") assert c.limit_txt == "~s" assert len(c.view) == 0 f.response = HTTPResponse.wrap(netlib.tutils.tresp()) c.update_flow(f) assert len(c.view) == 1 c.set_limit(None) assert len(c.view) == 1 f = tutils.tflow() c.add_flow(f) assert len(c.view) == 2 c.set_limit("~q") assert len(c.view) == 1 c.set_limit("~s") assert len(c.view) == 1 assert "Invalid" in c.set_limit("~")
def test_server_playback(self): s = flow.State() f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) fm.start_server_playback(pb, False, [], False, False, None, False, None, False) assert fm.do_server_playback(tutils.tflow()) fm.start_server_playback(pb, False, [], True, False, None, False, None, False) r = tutils.tflow() r.request.content = "gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) fm.start_server_playback(pb, False, [], True, False, None, False, None, False) q = Queue.Queue() fm.tick(q, 0) assert fm.should_exit.is_set() fm.stop_server_playback() assert not fm.server_playback
def test_refresh_cookie(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) # Invalid expires format, sent to us by Reddit. c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/" assert r._refresh_cookie(c, 60) c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure" assert "00:21:38" in r._refresh_cookie(c, 60)
def test_backup(self): f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp()) f.request.content = "foo" assert not f.modified() f.backup() f.request.content = "bar" assert f.modified() f.revert() assert f.request.content == "foo"
def test_missing_content(self): cs = StringIO() o = dump.Options(flow_detail=3) m = dump.DumpMaster(None, o, outfile=cs) f = tutils.tflow() f.request.content = CONTENT_MISSING m.handle_request(f) f.response = HTTPResponse.wrap(netlib.tutils.tresp()) f.response.content = CONTENT_MISSING m.handle_response(f) assert "content missing" in cs.getvalue()
def _cycle(self, m, content): f = tutils.tflow(req=netlib.tutils.treq(content=content)) l = Log("connect") l.reply = mock.MagicMock() m.handle_log(l) m.handle_clientconnect(f.client_conn) m.handle_serverconnect(f.server_conn) m.handle_request(f) f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=content)) f = m.handle_response(f) m.handle_clientdisconnect(f.client_conn) return f
def test_strfuncs(): t = HTTPResponse.wrap(netlib.tutils.tresp()) t.is_replay = True dump.str_response(t) f = tutils.tflow() f.client_conn = None f.request.stickycookie = True assert "stickycookie" in dump.str_request(f, False) assert "stickycookie" in dump.str_request(f, True) assert "replay" in dump.str_request(f, False) assert "replay" in dump.str_request(f, True)
def _cycle(self, m, content): f = tutils.tflow(req=netlib.tutils.treq(body=content)) l = Log("connect") l.reply = mock.MagicMock() m.handle_log(l) m.handle_clientconnect(f.client_conn) m.handle_serverconnect(f.server_conn) m.handle_request(f) f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=content)) f = m.handle_response(f) m.handle_clientdisconnect(f.client_conn) return f
def expediaRequest(context, flow): if flow.request.method == "GET": # handle the https redirect # 1) Find the hashParam by doing an https request with the user's headers hdrs = {} for h in flow.request.headers: if h not in filterHeaders: hdrs[h] = flow.request.headers[h] # else: # lg.write(" -- not using\n") c = httplib.HTTPSConnection("www.expedia.com") params = urllib.urlencode({ 'destination': flow.request.query["destination"], 'startDate': flow.request.query["startDate"], 'endDate': flow.request.query["endDate"], 'adults': flow.request.query["adults"] }) c.request("GET", "/Hotel-Search?#%s" % params, "", hdrs) response = c.getresponse() data = None if "gzip" == response.getheader('content-encoding'): #response.info().get("Content-encoding"): buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = response.read() m = RE_EXPEDIA_HASHPARAM.search(data) hashParam = m.group(1) #lg.write("hashParam:%s\n" % hashParam) lg.flush() h = Headers(Content_Type="text/plain") resp = HTTPResponse("HTTP/1.1", 200, "OK", h, hashParam) flow.reply(resp) # flow.request.path = 'http://www.expedia.com/vspersonal/Hotel-Search?inpAjax=true&responsive=true' if flow.request.method == "POST": if flow.request.urlencoded_form['hashParam'][ 0] == "f47b011acfc5249e9966c1acd0c52c9d163daae5": flow.request.headers[ 'Cookie'] = 'SSID1=CACvtx1wAAAAAABYJY9W-d5FBFglj1YBAAAAAAAAAAAAWCWPVgAKircEAAEXdwAAWCWPVgEAsAQAAa92AABYJY9WAQC5BAABJXcAAFglj1YBAK8EAAGsdgAAWCWPVgEAtQQAAQZ3AABYJY9WAQC0BAABA3cAAFglj1YBALgEAAEadwAAWCWPVgEAvgQAAcJ3AABYJY9WAQA; SSSC1=1.G6237245068890463993.1|1199.30380:1200.30383:1204.30467:1205.30470:1207.30487:1208.30490:1209.30501:1214.30658; MC1=GUID=803cd47a399c45cb957978f8b8aab687; JSESSION=3440b793-4d5f-4e0e-b691-0ae1fdad4691; tpid=v.1,1; iEAPID=0,; abucket=CgAUc1aPJVeT6BB7pl0ZAg==; SSRT1=WCWPVgIDAQ; __utmt=1; __utma=16308457.2088891848.1452221786.1452221786.1452221786.1; __utmb=16308457.1.10.1452221786; __utmc=16308457; __utmz=16308457.1452221786.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aspp=v.1,0|||||||||||||; eid=-1; s_cc=true; ipsnf3=v.3|us|1|744|honolulu; _cc=ZDA1Y2JhMzgtNjU2Ni00Mjc5LWE0YmUtOTkzZWJjZGM5Y2MzOjE0NTIyMjE3ODgwNjg; MediaCookie=0%7C2593%2C2563%2CBKC%2C31127; utag_main=v_id:01521f29ee8a0074022bed0e89f00b07900180710093c$_sn:1$_ss:1$_pn:1%3Bexp-session$_st:1452223587786$ses_id:1452221787786%3Bexp-session$dc_visit:1$dc_event:1%3Bexp-session; _ga=GA1.2.2088891848.1452221786; _gat_ua=1; __qca=P0-1655839093-1452221791945; _tq_id.TV-721872-1.7ec4=f13e60f0bf8f7421.1452221792.0.1452221792..; __gads=ID=e467f079dd328a46:T=1452221792:S=ALNI_MbWgX3mjx_B4xkNLTpIppiKCSyQYg; IAID=418047a9-b7c1-4472-95cd-0a75bfca38aa; s_fid=26BBFDB29B009DC3-29429116A147DDF7; cesc=%7B%7D; s_vi=[CS]v1|2B4792AE05013A9E-60001607E0009084[CE]; SSLB=1; linfo=v.4,|0|0|255|1|0||||||||1033|0|0||0|0|0|-1|-1' flow.request.path = flow.request.path.replace('/vspersonal', '')
def request(context, flow): # pretty_host takes the "Host" header of the request into account, # which is useful in transparent mode where we usually only have the IP # otherwise. # Method 1: Answer with a locally generated response if flow.request.pretty_host.endswith("example.com"): resp = HTTPResponse("HTTP/1.1", 200, "OK", Headers(Content_Type="text/html"), "helloworld") flow.reply(resp) # Method 2: Redirect the request to a different server if flow.request.pretty_host.endswith("example.org"): flow.request.host = "mitmproxy.org"
def test_refresh_cookie(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) # Invalid expires format, sent to us by Reddit. c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/" assert r._refresh_cookie(c, 60) c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure" assert "00:21:38" in r._refresh_cookie(c, 60) # https://github.com/mitmproxy/mitmproxy/issues/773 c = ">=A" with tutils.raises(ValueError): r._refresh_cookie(c, 60)
def test_server_playback_kill(self): s = flow.State() f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True fm.start_server_playback(pb, True, [], False, False, None, False, None, False) f = tutils.tflow() f.request.host = "nonexistent" fm.process_new_request(f) assert "killed" in f.error.msg
def test_refresh(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) n = time.time() r.headers["date"] = email.utils.formatdate(n) pre = r.headers["date"] r.refresh(n) assert pre == r.headers["date"] r.refresh(n + 60) d = email.utils.parsedate_tz(r.headers["date"]) d = email.utils.mktime_tz(d) # Weird that this is not exact... assert abs(60 - (d - n)) <= 1 r.headers["set-cookie"] = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure" r.refresh()
def request(context, flow): # pretty_host(hostheader=True) takes the Host: header of the request into account, # which is useful in transparent mode where we usually only have the IP # otherwise. # Method 1: Answer with a locally generated response if flow.request.pretty_host(hostheader=True).endswith("example.com"): resp = HTTPResponse([1, 1], 200, "OK", ODictCaseless([["Content-Type", "text/html"]]), "helloworld") flow.reply(resp) # Method 2: Redirect the request to a different server if flow.request.pretty_host(hostheader=True).endswith("example.org"): flow.request.host = "mitmproxy.org" flow.request.update_host_header()
def request(context, flow): # reject almost every url if flow.request.host not in allowed_hosts: context.kill_flow(flow) lg.write("Killed flow to %s\n" % flow.request.host) lg.flush() return # else: # lg.write("Allowed flow to %s\n" % flow.request.host) # lg.flush() # reject if it doesn't say /vspersonal/xxx if '/vspersonal' not in flow.request.path: context.kill_flow(flow) lg.write("Killed flow with path %s\n" % flow.request.path) lg.flush() return # fix for FF OPTIONS: http://stackoverflow.com/questions/1099787/jquery-ajax-post-sending-options-as-request-method-in-firefox if flow.request.method == "OPTIONS": lg.write("Options Req to %s\n" % flow.request.host) lg.flush() try: h = Headers(Content_Type="text/plain") # h['Access-Control-Allow-Origin'] = '*' # didn't work for some reason h['Access-Control-Allow-Origin'] = flow.request.headers[ 'Origin'] # allow where they're coming from, consider only allowing *.volunteerscience.com h['Access-Control-Allow-Methods'] = 'POST, GET, OPTIONS' h['Access-Control-Max-Age'] = '60000' # one min h['Access-Control-Allow-Headers'] = 'origin, x-csrftoken, content-type, accept' h['Access-Control-Allow-Credentials'] = 'true' # required for xhrFields: { withCredentials: true } resp = HTTPResponse("HTTP/1.1", 200, "OK", h, "") flow.reply(resp) except: traceback.print_exc(file=lg) lg.write("Options Req Complete to %s\n" % flow.request.host) lg.flush() return if flow.request.host == "www.expedia.com": return expediaRequest(context, flow)
def handle_request(self, flow): if "TransformToRun" in flow.request.url: req_headers = dict(flow.request.headers.fields) req_url = flow.request.url req_data = flow.request.data.content m = MaltegoMsg(req_data) TRX = MaltegoTransform() if not m.getProperty("_uid"): uid = str(uuid.uuid4()) NewEnt = TRX.addEntity(m.Type, m.Value) for k, v in m.Properties.iteritems(): NewEnt.addProperty(k, k, "nostrict", v) NewEnt.addProperty("_uid", "_uid", "nostrict", uid) #NewEnt.setNote(uid) data = TRX.returnOutput() #Add to Queue future = self.Session.post(req_url, headers=req_headers, data=req_data) self.futures[uid] = future else: #Check status of request uid = m.getProperty("_uid") futReq = self.futures.get(uid) if futReq and futReq.done(): del self.futures[uid] data = futReq.result().text else: data = TRX.returnOutput() resp = HTTPResponse("HTTP/1.1", 200, "OK", Headers(Content_Type="text/xml;charset=UTF-8"), data) flow.reply(resp)
def test_all(self): s = flow.State() fm = flow.FlowMaster(None, s) fm.anticache = True fm.anticomp = True f = tutils.tflow(req=None) fm.handle_clientconnect(f.client_conn) f.request = HTTPRequest.wrap(netlib.tutils.treq()) fm.handle_request(f) assert s.flow_count() == 1 f.response = HTTPResponse.wrap(netlib.tutils.tresp()) fm.handle_response(f) assert not fm.handle_response(None) assert s.flow_count() == 1 fm.handle_clientdisconnect(f.client_conn) f.error = Error("msg") f.error.reply = controller.DummyReply() fm.handle_error(f) fm.load_script(tutils.test_data.path("scripts/a.py")) fm.shutdown()
def handle_request(self, f): resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.content = CONTENT_MISSING f.reply(resp)
def handle_request(self, f): resp = HTTPResponse.wrap(netlib.tutils.tresp()) f.reply(resp)
def test_get_content_type(self): h = odict.ODictCaseless() h["Content-Type"] = ["text/plain"] resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = h assert resp.headers.get_first("content-type") == "text/plain"
def response(code, body="", headers=Headers()): return HTTPResponse("HTTP/1.1", code, status_message(code), headers, body)
def test_get_content_type(self): resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = Headers(content_type="text/plain") assert resp.headers["content-type"] == "text/plain"
def _add_response(self, state): f = tutils.tflow() state.add_flow(f) f.response = HTTPResponse.wrap(netlib.tutils.tresp()) state.update_flow(f)