def test_get_content_view():
desc, lines, err = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
)
assert "Raw" in desc
assert list(lines)
assert not err
desc, lines, err = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert desc == "JSON"
desc, lines, err = cv.get_content_view(
cv.get("JSON"),
b"[1, 2",
)
assert "Couldn't parse" in desc
with mock.patch("mitmproxy.contentviews.ViewAuto.__call__") as view_auto:
view_auto.side_effect = ValueError
desc, lines, err = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
)
assert err
assert "Couldn't parse" in desc
def _echo_message(self, message):
if self.o.flow_detail >= 2:
headers = "\r\n".join(
"{}: {}".format(
click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True),
click.style(strutils.bytes_to_escaped_str(v), fg="blue"))
for k, v in message.headers.fields
)
self.echo(headers, indent=4)
if self.o.flow_detail >= 3:
if message.content is None:
self.echo("(content missing)", indent=4)
elif message.content:
self.echo("")
try:
type, lines = contentviews.get_content_view(
contentviews.get("Auto"),
message.content,
headers=message.headers
)
except exceptions.ContentViewException:
s = "Content viewer failed: \n" + traceback.format_exc()
self.add_event(s, "debug")
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
message.content,
headers=message.headers
)
styles = dict(
highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green")
)
def colorful(line):
yield u" " # we can already indent here
for (style, text) in line:
yield click.style(text, **styles.get(style, {}))
if self.o.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
lines_to_echo = list(lines_to_echo)
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines_to_echo
)
self.echo(content)
if next(lines, None):
self.echo("(cut off)", indent=4, dim=True)
if self.o.flow_detail >= 2:
self.echo("")
def handle_response(self, f):
flow.FlowMaster.handle_response(self, f)
if f:
f.reply()
if f.request.path.startswith('/api.php?action=get_player_data'):
print("Got monster data, processing...")
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_data.txt', 'w')
cap.write(content)
cap.close()
self.monster_data = content
if f.request.path.startswith('/api.php?action=get_user_mail'):
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_mail.txt', 'w')
cap.write(content)
cap.close()
self.mailbox_data = content
print("Got mail data, processing...")
if self.mailbox_data and self.monster_data:
thread.start_new_thread(lambda x,y,z: (update_padherder(x) and None) or (update_mails(y) and None) or z(),(self.monster_data,self.mailbox_data,self.reset_data))
return f
def clearall(self):
self.master.killextra = False
self.master.showhost = False
self.master.refresh_server_playback = True
self.master.server.config.no_upstream_cert = False
self.master.setheaders.clear()
self.master.replacehooks.clear()
self.master.set_ignore_filter([])
self.master.set_tcp_filter([])
self.master.options.update(
scripts = [],
anticache = False,
anticomp = False,
stickyauth = None,
stickycookie = None
)
self.master.state.default_body_view = contentviews.get("Auto")
signals.update_settings.send(self)
signals.status_message.send(
message = "All select.Options cleared",
expire = 1
)
def test_get_message_content_view():
r = netlib.tutils.treq()
desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
assert desc == "Raw"
r.encode("gzip")
desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
assert desc == "[decoded gzip] Raw"
r.headers["content-encoding"] = "deflate"
desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
assert desc == "[cannot decode] Raw"
r.content = None
desc, lines, err = cv.get_message_content_view(cv.get("Raw"), r)
assert list(lines) == [[("error", "content missing")]]
def __init__(self):
flow.State.__init__(self)
self.focus = None
self.follow_focus = None
self.default_body_view = contentviews.get("Auto")
self.flowsettings = weakref.WeakKeyDictionary()
self.last_search = None
def __call__(self, data, **metadata):
headers = metadata.get("headers", {})
ctype = headers.get("content-type")
if data and ctype:
ct = http.parse_content_type(ctype) if ctype else None
ct = "%s/%s" % (ct[0], ct[1])
if ct in contentviews.content_types_map:
return contentviews.content_types_map[ct][0](data, **metadata)
elif strutils.is_xml(data):
return contentviews.get("XML/HTML")(data, **metadata)
if metadata.get("query"):
return contentviews.get("Query")(data, **metadata)
if data and strutils.is_mostly_bin(data):
return contentviews.get("Hex")(data)
if not data:
return "No content", []
return contentviews.get("Raw")(data)
def test_custom_views():
class ViewNoop(cv.View):
name = "noop"
prompt = ("noop", "n")
content_types = ["text/none"]
def __call__(self, data, **metadata):
return "noop", cv.format_text(data)
view_obj = ViewNoop()
cv.add(view_obj)
assert cv.get("noop")
r = cv.get_content_view(
cv.get("noop"),
"[1, 2, 3]",
headers=Headers(
content_type="text/plain"
)
)
assert "noop" in r[0]
# now try content-type matching
r = cv.get_content_view(
cv.get("Auto"),
"[1, 2, 3]",
headers=Headers(
content_type="text/none"
)
)
assert "noop" in r[0]
# now try removing the custom view
cv.remove(view_obj)
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(
content_type="text/none"
)
)
assert "noop" not in r[0]
def test_get_content_view(self):
r = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert r[0] == "JSON"
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/vnd.api+json")
)
assert r[0] == "JSON"
tutils.raises(
ContentViewException,
cv.get_content_view,
cv.get("AMF"),
b"[1, 2",
headers=Headers()
)
r = cv.get_content_view(
cv.get("Auto"),
encoding.encode('gzip', b"[1, 2, 3]"),
headers=Headers(
content_type="application/json",
content_encoding="gzip"
)
)
assert "decoded gzip" in r[0]
assert "JSON" in r[0]
r = cv.get_content_view(
cv.get("XML"),
encoding.encode('gzip', b"[1, 2, 3]"),
headers=Headers(
content_type="application/json",
content_encoding="gzip"
)
)
assert "decoded gzip" in r[0]
assert "Raw" in r[0]
def get(self, flow_id, message, content_view):
message = getattr(self.flow, message)
description, lines, error = contentviews.get_message_content_view(
contentviews.get(content_view.replace('_', ' ')), message
)
# if error:
# add event log
self.write(dict(
lines=list(lines),
description=description
))
def _echo_message(self, message):
if self.flow_detail >= 2 and hasattr(message, "headers"):
headers = "\r\n".join(
"{}: {}".format(
click.style(
strutils.bytes_to_escaped_str(k), fg="blue", bold=True
),
click.style(
strutils.bytes_to_escaped_str(v), fg="blue"
)
)
for k, v in message.headers.fields
)
self.echo(headers, ident=4)
if self.flow_detail >= 3:
_, lines, error = contentviews.get_message_content_view(
contentviews.get("Auto"),
message
)
if error:
ctx.log.debug(error)
styles = dict(
highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green")
)
def colorful(line):
yield u" " # we can already indent here
for (style, text) in line:
yield click.style(text, **styles.get(style, {}))
if self.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines_to_echo
)
if content:
self.echo("")
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if self.flow_detail >= 2:
self.echo("")
def _get_content_view(self, viewmode, message, max_lines, _):
try:
query = None
if isinstance(message, models.HTTPRequest):
query = message.query
description, lines = contentviews.get_content_view(
viewmode, message.content, headers=message.headers, query=query
)
except exceptions.ContentViewException:
s = "Content viewer failed: \n" + traceback.format_exc()
signals.add_event(s, "error")
description, lines = contentviews.get_content_view(
contentviews.get("Raw"), message.content, headers=message.headers
)
description = description.replace("Raw", "Couldn't parse: falling back to Raw")
# Give hint that you have to tab for the response.
if description == "No content" and isinstance(message, models.HTTPRequest):
description = "No request content (press tab to view response)"
# If the users has a wide terminal, he gets fewer lines; this should not be an issue.
chars_per_line = 80
max_chars = max_lines * chars_per_line
total_chars = 0
text_objects = []
for line in lines:
txt = []
for (style, text) in line:
if total_chars + len(text) > max_chars:
text = text[:max_chars - total_chars]
txt.append((style, text))
total_chars += len(text)
if total_chars == max_chars:
break
# round up to the next line.
total_chars = int(math.ceil(total_chars / chars_per_line) * chars_per_line)
text_objects.append(urwid.Text(txt))
if total_chars == max_chars:
text_objects.append(urwid.Text([
("highlight", "Stopped displaying data after %d lines. Press " % max_lines),
("key", "f"),
("highlight", " to load all data.")
]))
break
return description, text_objects
def test_get_content_view(self):
r = cv.get_content_view(
cv.get("Raw"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/json")
)
assert r[0] == "JSON"
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2",
headers=Headers(content_type="application/json")
)
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
b"[1, 2, 3]",
headers=Headers(content_type="application/vnd.api+json")
)
assert r[0] == "JSON"
tutils.raises(
ContentViewException,
cv.get_content_view,
cv.get("AMF"),
b"[1, 2",
headers=Headers()
)
def clearall(self):
self.master.options.update(
anticache=False,
anticomp=False,
ignore_hosts=(),
tcp_hosts=(),
kill=False,
no_upstream_cert=False,
refresh_server_playback=True,
replacements=[],
scripts=[],
setheaders=[],
showhost=False,
stickyauth=None,
stickycookie=None,
)
self.master.state.default_body_view = contentviews.get("Auto")
signals.update_settings.send(self)
signals.status_message.send(message="All select.Options cleared", expire=1)
def clearall(self):
self.master.options.update(
anticache=False,
anticomp=False,
ignore_hosts=(),
tcp_hosts=(),
replay_kill_extra=False,
no_upstream_cert=False,
refresh_server_playback=True,
replacements=[],
scripts=[],
setheaders=[],
showhost=False,
stickyauth=None,
stickycookie=None,
)
self.master.state.default_body_view = contentviews.get("Auto")
signals.update_settings.send(self)
signals.status_message.send(message="All select.Options cleared",
expire=1)
def test_get_content_view(self):
r = cv.get_content_view(
cv.get("Raw"),
"[1, 2, 3]",
headers=Headers(content_type="application/json"))
assert "Raw" in r[0]
r = cv.get_content_view(
cv.get("Auto"),
"[1, 2, 3]",
headers=Headers(content_type="application/json"))
assert r[0] == "JSON"
r = cv.get_content_view(
cv.get("Auto"),
"[1, 2",
headers=Headers(content_type="application/json"))
assert "Raw" in r[0]
tutils.raises(ContentViewException,
cv.get_content_view,
cv.get("AMF"),
"[1, 2",
headers=Headers())
r = cv.get_content_view(cv.get("Auto"),
encoding.encode('gzip', "[1, 2, 3]"),
headers=Headers(
content_type="application/json",
content_encoding="gzip"))
assert "decoded gzip" in r[0]
assert "JSON" in r[0]
r = cv.get_content_view(cv.get("XML"),
encoding.encode('gzip', "[1, 2, 3]"),
headers=Headers(
content_type="application/json",
content_encoding="gzip"))
assert "decoded gzip" in r[0]
assert "Raw" in r[0]
def test_custom_contentviews(self):
m, sc = tscript("custom_contentviews.py")
pig = contentviews.get("pig_latin_HTML")
_, fmt = pig(b"<html>test!</html>")
assert any(b'esttay!' in val[0][1] for val in fmt)
assert not pig(b"gobbledygook")
def handle_response(self, f):
flow.FlowMaster.handle_response(self, f)
if f:
f.reply()
if f.request.path.startswith('/api.php?action=get_player_data'):
evt = custom_events.wxStatusEvent(message="Got box data, processing...")
wx.PostEvent(self.status_ctrl,evt)
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_data.txt', 'w')
cap.write(content)
cap.close()
thread.start_new_thread(padherder_sync.do_sync, (content, self.status_ctrl, self.region))
elif f.request.path.startswith('/api.php?action=get_user_mail'):
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_mail.txt', 'w')
cap.write(content)
cap.close()
mails = parse_mail(content)
mails.reverse()
evt = custom_events.wxMailEvent(mails=mails)
wx.PostEvent(self.mail_tab, evt)
evt = custom_events.wxStatusEvent(message="Got mail data, processing...")
wx.PostEvent(self.status_ctrl,evt)
else:
config = wx.ConfigBase.Get()
actions = config.Read("customcapture")
if actions != "" and actions != None:
for act in actions.split(','):
act = act.strip()
if f.request.path.startswith('/api.php?action=%s' % act):
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_%s.txt' % act, 'w')
cap.write(content)
cap.close()
evt = custom_events.wxStatusEvent(message="Got custom capture %s" % act)
wx.PostEvent(self.status_ctrl,evt)
return f
def _echo_message(self, message):
if self.flow_detail >= 2 and hasattr(message, "headers"):
headers = "\r\n".join(
"{}: {}".format(
click.style(
strutils.bytes_to_escaped_str(k), fg="blue", bold=True
),
click.style(
strutils.bytes_to_escaped_str(v), fg="blue"
)
)
for k, v in message.headers.fields
)
self.echo(headers, ident=4)
if self.flow_detail >= 3:
try:
content = message.content
except ValueError:
content = message.get_content(strict=False)
if content is None:
self.echo("(content missing)", ident=4)
elif content:
self.echo("")
try:
_, lines = contentviews.get_content_view(
contentviews.get("Auto"),
content,
headers=getattr(message, "headers", None)
)
except exceptions.ContentViewException:
s = "Content viewer failed: \n" + traceback.format_exc()
ctx.log.debug(s)
_, lines = contentviews.get_content_view(
contentviews.get("Raw"),
content,
headers=getattr(message, "headers", None)
)
styles = dict(
highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green")
)
def colorful(line):
yield u" " # we can already indent here
for (style, text) in line:
yield click.style(text, **styles.get(style, {}))
if self.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
lines_to_echo = list(lines_to_echo)
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines_to_echo
)
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if self.flow_detail >= 2:
self.echo("")
def handle_response(self, f):
flow.FlowMaster.handle_response(self, f)
if f:
f.reply()
if f.request.path.startswith('/api.php?action=get_player_data'):
evt = custom_events.wxStatusEvent(message="Got box data, processing...")
wx.PostEvent(self.status_ctrl,evt)
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_data.txt', 'w')
cap.write(content)
cap.close()
thread.start_new_thread(padherder_sync.do_sync, (content, self.status_ctrl, self.region))
elif f.request.path.startswith('/api.php?action=get_user_mail'):
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_mail.txt', 'w')
cap.write(content)
cap.close()
mails = parse_mail(content)
mails.reverse()
evt = custom_events.wxMailEvent(mails=mails)
wx.PostEvent(self.mail_tab, evt)
evt = custom_events.wxStatusEvent(message="Got mail data, processing...")
wx.PostEvent(self.status_ctrl,evt)
else:
config = wx.ConfigBase.Get()
actions = config.Read("customcapture")
if actions != "" and actions != None:
for act in actions.split(','):
act = act.strip()
if f.request.path.startswith('/api.php?action=%s' % act):
resp = f.response.content
type, lines = contentviews.get_content_view(
contentviews.get("Raw"),
f.response.content,
headers=f.response.headers)
def colorful(line):
for (style, text) in line:
yield text
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines
)
cap = open('captured_%s.txt' % act, 'w')
cap.write(content)
cap.close()
evt = custom_events.wxStatusEvent(message="Got custom capture %s" % act)
wx.PostEvent(self.status_ctrl,evt)
return f
def _get_content_view(self, message, viewmode, max_lines, _):
try:
content = message.content
if content != message.raw_content:
enc = "[decoded {}]".format(
message.headers.get("content-encoding"))
else:
enc = None
except ValueError:
content = message.raw_content
enc = "[cannot decode]"
try:
query = None
if isinstance(message, models.HTTPRequest):
query = message.query
description, lines = contentviews.get_content_view(
viewmode, content, headers=message.headers, query=query)
except exceptions.ContentViewException:
s = "Content viewer failed: \n" + traceback.format_exc()
signals.add_log(s, "error")
description, lines = contentviews.get_content_view(
contentviews.get("Raw"), content, headers=message.headers)
description = description.replace(
"Raw", "Couldn't parse: falling back to Raw")
if enc:
description = " ".join([enc, description])
# Give hint that you have to tab for the response.
if description == "No content" and isinstance(message,
models.HTTPRequest):
description = "No request content (press tab to view response)"
# If the users has a wide terminal, he gets fewer lines; this should not be an issue.
chars_per_line = 80
max_chars = max_lines * chars_per_line
total_chars = 0
text_objects = []
for line in lines:
txt = []
for (style, text) in line:
if total_chars + len(text) > max_chars:
text = text[:max_chars - total_chars]
txt.append((style, text))
total_chars += len(text)
if total_chars == max_chars:
break
# round up to the next line.
total_chars = int(
math.ceil(total_chars / chars_per_line) * chars_per_line)
text_objects.append(urwid.Text(txt))
if total_chars == max_chars:
text_objects.append(
urwid.Text([
("highlight",
"Stopped displaying data after %d lines. Press " %
max_lines), ("key", "f"),
("highlight", " to load all data.")
]))
break
return description, text_objects
def test_custom_contentviews(self):
m, sc = tscript("simple/custom_contentview.py")
swapcase = contentviews.get("swapcase")
_, fmt = swapcase(b"<html>Test!</html>")
assert any(b'tEST!' in val[0][1] for val in fmt)
def change_this_display_mode(self, t):
view = contentviews.get(t)
self.view.settings[self.flow][(self.tab_offset,
"prettyview")] = view.name.lower()
signals.flow_change.send(self, flow=self.flow)
def test_custom_contentviews(self, tdata):
with taddons.context() as tctx:
tctx.script(tdata.path("../examples/simple/custom_contentview.py"))
swapcase = contentviews.get("swapcase")
_, fmt = swapcase(b"<html>Test!</html>")
assert any(b'tEST!' in val[0][1] for val in fmt)
def test_custom_contentviews(self, tdata):
with taddons.context() as tctx:
tctx.script(tdata.path("../examples/addons/contentview.py"))
swapcase = contentviews.get("swapcase")
_, fmt = swapcase(b"<html>Test!</html>")
assert any(b'tEST!' in val[0][1] for val in fmt)
def test_custom_contentviews(self):
m, sc = tscript("custom_contentviews.py")
pig = contentviews.get("pig_latin_HTML")
_, fmt = pig(b"<html>test!</html>")
assert any(b'esttay!' in val[0][1] for val in fmt)
assert not pig(b"gobbledygook")
def test_custom_contentviews(self):
m, sc = tscript("simple/custom_contentview.py")
swapcase = contentviews.get("swapcase")
_, fmt = swapcase(b"<html>Test!</html>")
assert any(b'tEST!' in val[0][1] for val in fmt)
