def authenticate(self, request):
if self.config.authenticator:
if self.config.authenticator.authenticate(request.headers):
self.config.authenticator.clean(request.headers)
else:
self.send_response(make_error_response(
407,
"Proxy Authentication Required",
Headers(**self.config.authenticator.auth_challenge_headers())
))
return False
return True
def test_set_all(self):
headers = Headers(Host="example.com")
headers.set_all("Accept", ["text/plain"])
assert len(headers) == 2
assert "accept" in headers
headers = self._2host()
headers.set_all("Host", ["example.org"])
assert headers["host"] == "example.org"
headers.set_all("Host", ["example.org", "example.net"])
assert headers["host"] == "example.org, example.net"
def test_modify_form(self):
m, sc = tscript("modify_form.py")
form_header = Headers(content_type="application/x-www-form-urlencoded")
f = tutils.tflow(req=netutils.treq(headers=form_header))
m.request(f)
assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks"
f.request.headers["content-type"] = ""
m.request(f)
assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")]
def test_get_cookies_twocookies(self):
resp = tresp()
resp.headers = Headers([
[b"Set-Cookie", b"cookiename=cookievalue"],
[b"Set-Cookie", b"othercookie=othervalue"]
])
result = resp.cookies
assert len(result) == 2
assert "cookiename" in result
assert result["cookiename"] == ("cookievalue", CookieAttrs())
assert "othercookie" in result
assert result["othercookie"] == ("othervalue", CookieAttrs())
def test_init(self):
headers = Headers()
assert len(headers) == 0
headers = Headers([[b"Host", b"example.com"]])
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(Host="example.com")
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"]],
Host="example.com"
)
assert len(headers) == 1
assert headers["Host"] == "example.com"
headers = Headers(
[[b"Host", b"invalid"], [b"Accept", b"text/plain"]],
Host="example.com"
)
assert len(headers) == 2
assert headers["Host"] == "example.com"
assert headers["Accept"] == "text/plain"
with raises(ValueError):
Headers([[b"Host", u"not-bytes"]])
def expediaRequest(context, flow):
if flow.request.method == "GET": # handle the https redirect
# 1) Find the hashParam by doing an https request with the user's headers
hdrs = {}
for h in flow.request.headers:
if h not in filterHeaders:
hdrs[h] = flow.request.headers[h]
# else:
# lg.write(" -- not using\n")
c = httplib.HTTPSConnection("www.expedia.com")
params = urllib.urlencode({
'destination':
flow.request.query["destination"],
'startDate':
flow.request.query["startDate"],
'endDate':
flow.request.query["endDate"],
'adults':
flow.request.query["adults"]
})
c.request("GET", "/Hotel-Search?#%s" % params, "", hdrs)
response = c.getresponse()
data = None
if "gzip" == response.getheader('content-encoding'):
#response.info().get("Content-encoding"):
buf = StringIO(response.read())
f = gzip.GzipFile(fileobj=buf)
data = f.read()
else:
data = response.read()
m = RE_EXPEDIA_HASHPARAM.search(data)
hashParam = m.group(1)
#lg.write("hashParam:%s\n" % hashParam)
lg.flush()
h = Headers(Content_Type="text/plain")
resp = HTTPResponse("HTTP/1.1", 200, "OK", h, hashParam)
flow.reply(resp)
# flow.request.path = 'http://www.expedia.com/vspersonal/Hotel-Search?inpAjax=true&responsive=true'
if flow.request.method == "POST":
if flow.request.urlencoded_form['hashParam'][
0] == "f47b011acfc5249e9966c1acd0c52c9d163daae5":
flow.request.headers[
'Cookie'] = 'SSID1=CACvtx1wAAAAAABYJY9W-d5FBFglj1YBAAAAAAAAAAAAWCWPVgAKircEAAEXdwAAWCWPVgEAsAQAAa92AABYJY9WAQC5BAABJXcAAFglj1YBAK8EAAGsdgAAWCWPVgEAtQQAAQZ3AABYJY9WAQC0BAABA3cAAFglj1YBALgEAAEadwAAWCWPVgEAvgQAAcJ3AABYJY9WAQA; SSSC1=1.G6237245068890463993.1|1199.30380:1200.30383:1204.30467:1205.30470:1207.30487:1208.30490:1209.30501:1214.30658; MC1=GUID=803cd47a399c45cb957978f8b8aab687; JSESSION=3440b793-4d5f-4e0e-b691-0ae1fdad4691; tpid=v.1,1; iEAPID=0,; abucket=CgAUc1aPJVeT6BB7pl0ZAg==; SSRT1=WCWPVgIDAQ; __utmt=1; __utma=16308457.2088891848.1452221786.1452221786.1452221786.1; __utmb=16308457.1.10.1452221786; __utmc=16308457; __utmz=16308457.1452221786.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); aspp=v.1,0|||||||||||||; eid=-1; s_cc=true; ipsnf3=v.3|us|1|744|honolulu; _cc=ZDA1Y2JhMzgtNjU2Ni00Mjc5LWE0YmUtOTkzZWJjZGM5Y2MzOjE0NTIyMjE3ODgwNjg; MediaCookie=0%7C2593%2C2563%2CBKC%2C31127; utag_main=v_id:01521f29ee8a0074022bed0e89f00b07900180710093c$_sn:1$_ss:1$_pn:1%3Bexp-session$_st:1452223587786$ses_id:1452221787786%3Bexp-session$dc_visit:1$dc_event:1%3Bexp-session; _ga=GA1.2.2088891848.1452221786; _gat_ua=1; __qca=P0-1655839093-1452221791945; _tq_id.TV-721872-1.7ec4=f13e60f0bf8f7421.1452221792.0.1452221792..; __gads=ID=e467f079dd328a46:T=1452221792:S=ALNI_MbWgX3mjx_B4xkNLTpIppiKCSyQYg; IAID=418047a9-b7c1-4472-95cd-0a75bfca38aa; s_fid=26BBFDB29B009DC3-29429116A147DDF7; cesc=%7B%7D; s_vi=[CS]v1|2B4792AE05013A9E-60001607E0009084[CE]; SSLB=1; linfo=v.4,|0|0|255|1|0||||||||1033|0|0||0|0|0|-1|-1'
flow.request.path = flow.request.path.replace('/vspersonal', '')
def test_get_cookies_with_parameters(self):
resp = tresp()
resp.headers = Headers(set_cookie="cookiename=cookievalue;domain=example.com;expires=Wed Oct 21 16:29:41 2015;path=/; HttpOnly")
result = resp.cookies
assert len(result) == 1
assert "cookiename" in result
assert result["cookiename"][0] == "cookievalue"
attrs = result["cookiename"][1]
assert len(attrs) == 4
assert attrs["domain"] == "example.com"
assert attrs["expires"] == "Wed Oct 21 16:29:41 2015"
assert attrs["path"] == "/"
assert attrs["httponly"] is None
def test_with_body(self):
bytes = HTTP2Protocol(self.c, is_server=True).assemble_response(http.Response(
(2, 0),
200,
'',
Headers(foo="bar"),
'foobar'
))
assert len(bytes) == 2
assert bytes[0] ==\
'00000901040000000288408294e7838c767f'.decode('hex')
assert bytes[1] ==\
'000006000100000002666f6f626172'.decode('hex')
def create_request(self, method, scheme, host, port, path):
"""
this method creates a new artificial and minimalist request also adds it to flowlist
"""
c = ClientConnection.make_dummy(("", 0))
s = ServerConnection.make_dummy((host, port))
f = HTTPFlow(c, s)
headers = Headers()
req = HTTPRequest("absolute", method, scheme, host, port, path,
b"HTTP/1.1", headers, b"")
f.request = req
return self.load_flow(f)
def request(context, flow):
# pretty_host takes the "Host" header of the request into account,
# which is useful in transparent mode where we usually only have the IP
# otherwise.
# Method 1: Answer with a locally generated response
if flow.request.pretty_host.endswith("example.com"):
resp = HTTPResponse("HTTP/1.1", 200, "OK",
Headers(Content_Type="text/html"), "helloworld")
flow.reply(resp)
# Method 2: Redirect the request to a different server
if flow.request.pretty_host.endswith("example.org"):
flow.request.host = "mitmproxy.org"
def test_connection_close():
headers = Headers()
assert connection_close(b"HTTP/1.0", headers)
assert not connection_close(b"HTTP/1.1", headers)
headers["connection"] = "keep-alive"
assert not connection_close(b"HTTP/1.1", headers)
headers["connection"] = "close"
assert connection_close(b"HTTP/1.1", headers)
headers["connection"] = "foobar"
assert connection_close(b"HTTP/1.0", headers)
assert not connection_close(b"HTTP/1.1", headers)
def test_get_content_view(self):
r = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert r[0] == "JSON"
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/vnd.api+json")
)
assert r[0] == "JSON"
tutils.raises(
ContentViewException,
cv.get_content_view,
cv.get("AMF"),
b"[1, 2",
headers=Headers()
)
def test_delitem(self):
headers = Headers(Host="example.com")
assert len(headers) == 1
del headers["host"]
assert len(headers) == 0
try:
del headers["host"]
except KeyError:
assert True
else:
assert False
headers = self._2host()
del headers["Host"]
assert len(headers) == 0
def test_multipartdecode():
boundary = 'somefancyboundary'
headers = Headers(content_type='multipart/form-data; boundary=' + boundary)
content = ("--{0}\n"
"Content-Disposition: form-data; name=\"field1\"\n\n"
"value1\n"
"--{0}\n"
"Content-Disposition: form-data; name=\"field2\"\n\n"
"value2\n"
"--{0}--".format(boundary).encode())
form = utils.multipartdecode(headers, content)
assert len(form) == 2
assert form[0] == (b"field1", b"value1")
assert form[1] == (b"field2", b"value2")
def test_multipartdecode():
boundary = 'somefancyboundary'
headers = Headers(content_type='multipart/form-data; boundary=%s' %
boundary)
content = "--{0}\n" \
"Content-Disposition: form-data; name=\"field1\"\n\n" \
"value1\n" \
"--{0}\n" \
"Content-Disposition: form-data; name=\"field2\"\n\n" \
"value2\n" \
"--{0}--".format(boundary)
form = utils.multipartdecode(headers, content)
assert len(form) == 2
assert form[0] == ('field1', 'value1')
assert form[1] == ('field2', 'value2')
def test_expected_http_body_size():
# Expect: 100-continue
assert expected_http_body_size(
treq(headers=Headers(expect="100-continue", content_length="42"))
) == 0
# http://tools.ietf.org/html/rfc7230#section-3.3
assert expected_http_body_size(
treq(method=b"HEAD"),
tresp(headers=Headers(content_length="42"))
) == 0
assert expected_http_body_size(
treq(method=b"CONNECT"),
tresp()
) == 0
for code in (100, 204, 304):
assert expected_http_body_size(
treq(),
tresp(status_code=code)
) == 0
# chunked
assert expected_http_body_size(
treq(headers=Headers(transfer_encoding="chunked")),
) is None
# explicit length
for val in (b"foo", b"-7"):
with raises(HttpSyntaxException):
expected_http_body_size(
treq(headers=Headers(content_length=val))
)
assert expected_http_body_size(
treq(headers=Headers(content_length="42"))
) == 42
# no length
assert expected_http_body_size(
treq(headers=Headers())
) == 0
assert expected_http_body_size(
treq(headers=Headers()), tresp(headers=Headers())
) == -1
def request(context, flow):
# reject almost every url
if flow.request.host not in allowed_hosts:
context.kill_flow(flow)
lg.write("Killed flow to %s\n" % flow.request.host)
lg.flush()
return
# else:
# lg.write("Allowed flow to %s\n" % flow.request.host)
# lg.flush()
# reject if it doesn't say /vspersonal/xxx
if '/vspersonal' not in flow.request.path:
context.kill_flow(flow)
lg.write("Killed flow with path %s\n" % flow.request.path)
lg.flush()
return
# fix for FF OPTIONS: http://stackoverflow.com/questions/1099787/jquery-ajax-post-sending-options-as-request-method-in-firefox
if flow.request.method == "OPTIONS":
lg.write("Options Req to %s\n" % flow.request.host)
lg.flush()
try:
h = Headers(Content_Type="text/plain")
# h['Access-Control-Allow-Origin'] = '*' # didn't work for some reason
h['Access-Control-Allow-Origin'] = flow.request.headers[
'Origin'] # allow where they're coming from, consider only allowing *.volunteerscience.com
h['Access-Control-Allow-Methods'] = 'POST, GET, OPTIONS'
h['Access-Control-Max-Age'] = '60000' # one min
h['Access-Control-Allow-Headers'] = 'origin, x-csrftoken, content-type, accept'
h['Access-Control-Allow-Credentials'] = 'true' # required for xhrFields: { withCredentials: true }
resp = HTTPResponse("HTTP/1.1", 200, "OK", h, "")
flow.reply(resp)
except:
traceback.print_exc(file=lg)
lg.write("Options Req Complete to %s\n" % flow.request.host)
lg.flush()
return
if flow.request.host == "www.expedia.com":
return expediaRequest(context, flow)
def test_get_content_view(self):
r = cv.get_content_view(
cv.get("Raw"),
"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert r[0] == "JSON"
r = cv.get_content_view(
cv.get("Auto"),
"[1, 2",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
tutils.raises(
ContentViewException,
cv.get_content_view,
cv.get("AMF"),
"[1, 2",
headers=Headers()
)
r = cv.get_content_view(
cv.get("Auto"),
encoding.encode('gzip', "[1, 2, 3]"),
headers=Headers(
content_type="application/json",
content_encoding="gzip"
)
)
assert "decoded gzip" in r[0]
assert "JSON" in r[0]
r = cv.get_content_view(
cv.get("XML"),
encoding.encode('gzip', "[1, 2, 3]"),
headers=Headers(
content_type="application/json",
content_encoding="gzip"
)
)
assert "decoded gzip" in r[0]
assert "Raw" in r[0]
def handle_request(self, flow):
hid = (flow.request.host, flow.request.port)
host = flow.request.pretty_host
path = flow.request.path[1:]
if self.is_supported_api(host) and os.path.isfile(path):
print "Stubbing: %s/%s" % (host, path)
f = open(path, 'r')
resp = HTTPResponse("HTTP/1.1", 200, "OK",
Headers(Content_Type="text/json"), f.read())
f.close()
flow.reply(resp)
elif self.is_supported_host(host):
print "Should we capture ?: %s/%s" % (host, path)
flow.reply()
else:
flow.reply()
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
headers = Headers([header.values(settings) for header in self.headers])
body = self.body
if body:
body = body.string()
resp = http.Response(
b'HTTP/2.0',
self.status_code.string(),
b'',
headers,
body,
)
resp.stream_id = self.stream_id
self.rendered_values = settings.protocol.assemble(resp)
return self.rendered_values
def test_setitem(self):
headers = Headers()
headers["Host"] = "example.com"
assert "Host" in headers
assert "host" in headers
assert headers["Host"] == "example.com"
headers["host"] = "example.org"
assert "Host" in headers
assert "host" in headers
assert headers["Host"] == "example.org"
headers["accept"] = "text/plain"
assert len(headers) == 2
assert "Accept" in headers
assert "Host" in headers
headers = self._2host()
assert len(headers.fields) == 2
headers["Host"] = "example.com"
assert len(headers.fields) == 1
assert "Host" in headers
def handle_request(self, flow):
if "TransformToRun" in flow.request.url:
req_headers = dict(flow.request.headers.fields)
req_url = flow.request.url
req_data = flow.request.data.content
m = MaltegoMsg(req_data)
TRX = MaltegoTransform()
if not m.getProperty("_uid"):
uid = str(uuid.uuid4())
NewEnt = TRX.addEntity(m.Type, m.Value)
for k, v in m.Properties.iteritems():
NewEnt.addProperty(k, k, "nostrict", v)
NewEnt.addProperty("_uid", "_uid", "nostrict", uid)
#NewEnt.setNote(uid)
data = TRX.returnOutput()
#Add to Queue
future = self.Session.post(req_url,
headers=req_headers,
data=req_data)
self.futures[uid] = future
else:
#Check status of request
uid = m.getProperty("_uid")
futReq = self.futures.get(uid)
if futReq and futReq.done():
del self.futures[uid]
data = futReq.result().text
else:
data = TRX.returnOutput()
resp = HTTPResponse("HTTP/1.1", 200, "OK",
Headers(Content_Type="text/xml;charset=UTF-8"),
data)
flow.reply(resp)
def create_request(self, method, scheme, host, port, path):
"""
this method creates a new artificial and minimalist request also adds it to flowlist
"""
c = ClientConnection.from_state(
dict(address=dict(address=(host, port), use_ipv6=False),
clientcert=None))
s = ServerConnection.from_state(
dict(
address=dict(address=(host, port), use_ipv6=False),
state=[],
source_address=None,
# source_address=dict(address=(host, port), use_ipv6=False),
cert=None,
sni=host,
ssl_established=True))
f = HTTPFlow(c, s)
headers = Headers()
req = HTTPRequest("absolute", method, scheme, host, port, path,
b"HTTP/1.1", headers, None, None, None, None)
f.request = req
return self.load_flow(f)
def make_error_response(status_code, message, headers=None):
response = status_codes.RESPONSES.get(status_code, "Unknown")
body = """
<html>
<head>
<title>%d %s</title>
</head>
<body>%s</body>
</html>
""".strip() % (status_code, response, message)
if not headers:
headers = Headers(Server=version.NAMEVERSION,
Connection="close",
Content_Length=str(len(body)),
Content_Type="text/html")
return HTTPResponse(
b"HTTP/1.1",
status_code,
response,
headers,
body,
)
def test_view_auto(self):
v = cv.ViewAuto()
f = v(
"foo",
headers=Headers()
)
assert f[0] == "Raw"
f = v(
"<html></html>",
headers=Headers(content_type="text/html")
)
assert f[0] == "HTML"
f = v(
"foo",
headers=Headers(content_type="text/flibble")
)
assert f[0] == "Raw"
f = v(
"<xml></xml>",
headers=Headers(content_type="text/flibble")
)
assert f[0].startswith("XML")
f = v(
"",
headers=Headers()
)
assert f[0] == "No content"
f = v(
"",
headers=Headers(),
query=multidict.MultiDict([("foo", "bar")]),
)
assert f[0] == "Query"
def edit(self, part):
if self.tab_offset == TAB_REQ:
message = self.flow.request
else:
if not self.flow.response:
self.flow.response = models.HTTPResponse(
self.flow.request.http_version, 200, "OK", Headers(), "")
self.flow.response.reply = controller.DummyReply()
message = self.flow.response
self.flow.backup()
if message == self.flow.request and part == "c":
self.master.view_grideditor(
grideditor.CookieEditor(self.master,
message.cookies.items(multi=True),
self.set_cookies, message))
if message == self.flow.response and part == "c":
self.master.view_grideditor(
grideditor.SetCookieEditor(self.master,
message.cookies.items(multi=True),
self.set_setcookies, message))
if part == "r":
with models.decoded(message):
# Fix an issue caused by some editors when editing a
# request/response body. Many editors make it hard to save a
# file without a terminating newline on the last line. When
# editing message bodies, this can cause problems. For now, I
# just strip the newlines off the end of the body when we return
# from an editor.
c = self.master.spawn_editor(message.content or "")
message.content = c.rstrip("\n")
elif part == "f":
if not message.urlencoded_form and message.content:
signals.status_prompt_onekey.send(
prompt=
"Existing body is not a URL-encoded form. Clear and edit?",
keys=[
("yes", "y"),
("no", "n"),
],
callback=self.edit_form_confirm,
args=(message, ))
else:
self.edit_form(message)
elif part == "h":
self.master.view_grideditor(
grideditor.HeaderEditor(self.master, message.headers.fields,
self.set_headers, message))
elif part == "p":
p = message.path_components
self.master.view_grideditor(
grideditor.PathEditor(self.master, p, self.set_path_components,
message))
elif part == "q":
self.master.view_grideditor(
grideditor.QueryEditor(self.master,
message.query.items(multi=True),
self.set_query, message))
elif part == "u":
signals.status_prompt.send(prompt="URL",
text=message.url,
callback=self.set_url)
elif part == "m" and message == self.flow.request:
signals.status_prompt_onekey.send(prompt="Method",
keys=common.METHOD_OPTIONS,
callback=self.edit_method)
elif part == "o":
signals.status_prompt.send(prompt="Code",
text=str(message.status_code),
callback=self.set_resp_status_code)
elif part == "m" and message == self.flow.response:
signals.status_prompt.send(prompt="Message",
text=message.reason,
callback=self.set_resp_reason)
signals.flow_change.send(self, flow=self.flow)
def set_headers(self, fields, conn):
conn.headers = Headers(fields)
signals.flow_change.send(self, flow=self.flow)
def test_get_content_type(self):
resp = HTTPResponse.wrap(netlib.tutils.tresp())
resp.headers = Headers(content_type="text/plain")
assert resp.headers["content-type"] == "text/plain"
def dummy_response(self, flow):
resp = HTTPResponse("HTTP/1.1", 444, "Blocked",
Headers(Content_Type="text/html"),
"You got blocked by CDNReaper")
flow.reply(resp)
def respond(flow, content):
resp = HTTPResponse("HTTP/1.1", 200, "OK",
Headers(Content_Type="text/xml"),
content)
flow.reply(resp)
