def _echo_message(self, message):
_, lines, error = contentviews.get_message_content_view(
self.default_contentview,
message
)
if error:
ctx.log.debug(error)
if self.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
styles = dict(
highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green")
)
content = u"\r\n".join(
u"".join(colorful(line, styles)) for line in lines_to_echo
)
if content:
self.echo("")
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if self.flow_detail >= 2:
self.echo("")
def _echo_message(
self,
message: Union[http.Message, TCPMessage, WebSocketMessage],
flow: Union[http.HTTPFlow, TCPFlow]
):
_, lines, error = contentviews.get_message_content_view(
ctx.options.dumper_default_contentview,
message,
flow
)
if error:
ctx.log.debug(error)
if ctx.options.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
content = "\r\n".join(
"".join(self._colorful(line)) for line in lines_to_echo
)
if content:
self.echo("")
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if ctx.options.flow_detail >= 2:
self.echo("")
def test_get_message_content_view():
r = tutils.treq()
desc, lines, err = cv.get_message_content_view("raw", r)
assert desc == "Raw"
r.encode("gzip")
desc, lines, err = cv.get_message_content_view("raw", r)
assert desc == "[decoded gzip] Raw"
r.headers["content-encoding"] = "deflate"
desc, lines, err = cv.get_message_content_view("raw", r)
assert desc == "[cannot decode] Raw"
r.content = None
desc, lines, err = cv.get_message_content_view("raw", r)
assert list(lines) == [[("error", "content missing")]]
def save_flows_content(path: pathlib.Path,
flows: typing.Iterable[flow.Flow]) -> None:
for f in flows:
for m in ('request', 'response'):
message = getattr(f, m)
message_path = path / "flows" / f.id / m
os.makedirs(str(message_path / "content"), exist_ok=True)
with open(str(message_path / 'content.data'),
'wb') as content_file:
# don't use raw_content here as this is served with a default content type
if message:
content_file.write(message.content)
else:
content_file.write(b'No content.')
# content_view
t = time.time()
if message:
description, lines, error = contentviews.get_message_content_view(
'Auto', message, f)
else:
description, lines = 'No content.', []
if time.time() - t > 0.1:
ctx.log(
"Slow content view: {} took {}s".format(
description.strip(), round(time.time() - t, 1)),
"info")
with open(str(message_path / "content" / "Auto.json"),
"w") as content_view_file:
json.dump(dict(lines=list(lines), description=description),
content_view_file)
def _echo_message(self, message: Union[http.Message, TCPMessage,
WebSocketMessage],
flow: Union[http.HTTPFlow, TCPFlow, WebSocketFlow]):
_, lines, error = contentviews.get_message_content_view(
ctx.options.dumper_default_contentview, message, flow)
if error:
ctx.log.debug(error)
if ctx.options.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
styles = dict(highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green"))
content = "\r\n".join("".join(colorful(line, styles))
for line in lines_to_echo)
if content:
self.echo("")
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if ctx.options.flow_detail >= 2:
self.echo("")
def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None:
for f in flows:
for m in ('request', 'response'):
message = getattr(f, m)
message_path = path / "flows" / f.id / m
os.makedirs(str(message_path / "content"), exist_ok=True)
with open(str(message_path / 'content.data'), 'wb') as content_file:
# don't use raw_content here as this is served with a default content type
if message:
content_file.write(message.content)
else:
content_file.write(b'No content.')
# content_view
t = time.time()
if message:
description, lines, error = contentviews.get_message_content_view(
'Auto', message
)
else:
description, lines = 'No content.', []
if time.time() - t > 0.1:
ctx.log(
"Slow content view: {} took {}s".format(
description.strip(),
round(time.time() - t, 1)
),
"info"
)
with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file:
json.dump(
dict(lines=list(lines), description=description),
content_view_file
)
def view_websocket_messages(self):
flow = self.flow
assert isinstance(flow, http.HTTPFlow)
assert flow.websocket is not None
if not flow.websocket.messages:
return searchable.Searchable(
[urwid.Text(("highlight", "No messages."))])
viewmode = self.master.commands.call("console.flowview.mode")
widget_lines = []
for m in flow.websocket.messages:
_, lines, _ = contentviews.get_message_content_view(
viewmode, m, flow)
for line in lines:
if m.from_client:
line.insert(
0, ("from_client", f"{common.SYMBOL_FROM_CLIENT} "))
else:
line.insert(0,
("to_client", f"{common.SYMBOL_TO_CLIENT} "))
widget_lines.append(urwid.Text(line))
if flow.intercepted:
markup = widget_lines[-1].get_text()[0]
widget_lines[-1].set_text(("intercept", markup))
widget_lines.insert(
0, self._contentview_status_bar(viewmode.capitalize(), viewmode))
return searchable.Searchable(widget_lines)
def get(self, flow_id, message, content_view):
message = getattr(self.flow, message)
description, lines, error = contentviews.get_message_content_view(
content_view.replace('_', ' '), message, self.flow)
# if error:
# add event log
self.write(dict(lines=list(lines), description=description))
def get(self, flow_id, message, content_view):
message = getattr(self.flow, message)
description, lines, error = contentviews.get_message_content_view(
content_view.replace('_', ' '), message
)
# if error:
# add event log
self.write(dict(
lines=list(lines),
description=description
))
def _echo_message(self, message):
if self.flow_detail >= 2 and hasattr(message, "headers"):
headers = "\r\n".join(
"{}: {}".format(
click.style(
strutils.bytes_to_escaped_str(k), fg="blue", bold=True
),
click.style(
strutils.bytes_to_escaped_str(v), fg="blue"
)
)
for k, v in message.headers.fields
)
self.echo(headers, ident=4)
if self.flow_detail >= 3:
_, lines, error = contentviews.get_message_content_view(
contentviews.get("Auto"),
message
)
if error:
ctx.log.debug(error)
styles = dict(
highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green")
)
def colorful(line):
yield u" " # we can already indent here
for (style, text) in line:
yield click.style(text, **styles.get(style, {}))
if self.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines_to_echo
)
if content:
self.echo("")
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if self.flow_detail >= 2:
self.echo("")
def _echo_message(self, message):
if self.flow_detail >= 2 and hasattr(message, "headers"):
headers = "\r\n".join(
"{}: {}".format(
click.style(
strutils.bytes_to_escaped_str(k), fg="blue", bold=True
),
click.style(
strutils.bytes_to_escaped_str(v), fg="blue"
)
)
for k, v in message.headers.fields
)
self.echo(headers, ident=4)
if self.flow_detail >= 3:
_, lines, error = contentviews.get_message_content_view(
self.default_contentview,
message
)
if error:
ctx.log.debug(error)
styles = dict(
highlight=dict(bold=True),
offset=dict(fg="blue"),
header=dict(fg="green", bold=True),
text=dict(fg="green")
)
def colorful(line):
yield u" " # we can already indent here
for (style, text) in line:
yield click.style(text, **styles.get(style, {}))
if self.flow_detail == 3:
lines_to_echo = itertools.islice(lines, 70)
else:
lines_to_echo = lines
content = u"\r\n".join(
u"".join(colorful(line)) for line in lines_to_echo
)
if content:
self.echo("")
self.echo(content)
if next(lines, None):
self.echo("(cut off)", ident=4, dim=True)
if self.flow_detail >= 2:
self.echo("")
def _get_content_view(self, viewmode, max_lines, flowtype, _):
message = self._get_content_view_message
self._get_content_view_message = None
_message = message
if flowtype is FlowType.Request:
message = message.request
else:
message = message.response
description, lines, error = contentviews.get_message_content_view(
viewmode, _message, flowtype)
if error:
signals.add_log(error, "error")
# Give hint that you have to tab for the response.
if description == "No content" and isinstance(message,
http.HTTPRequest):
description = "No request content (press tab to view response)"
# If the users has a wide terminal, he gets fewer lines; this should not be an issue.
chars_per_line = 80
max_chars = max_lines * chars_per_line
total_chars = 0
text_objects = []
for line in lines:
txt = []
for (style, text) in line:
if total_chars + len(text) > max_chars:
text = text[:max_chars - total_chars]
txt.append((style, text))
total_chars += len(text)
if total_chars == max_chars:
break
# round up to the next line.
total_chars = int(
math.ceil(total_chars / chars_per_line) * chars_per_line)
text_objects.append(urwid.Text(txt))
if total_chars == max_chars:
text_objects.append(
urwid.Text([
("highlight",
"Stopped displaying data after %d lines. Press " %
max_lines), ("key", "f"),
("highlight", " to load all data.")
]))
break
return description, text_objects
def message_to_json(
self,
viewname: str,
message: Union[http.Message, TCPMessage, WebSocketMessage],
flow: Union[HTTPFlow, TCPFlow],
max_lines: Optional[int] = None
):
description, lines, error = contentviews.get_message_content_view(viewname, message, flow)
if error:
self.master.log.error(error)
if max_lines:
lines = islice(lines, max_lines)
return dict(
lines=list(lines),
description=description,
)
def view_websocket_messages(self):
flow = self.flow
assert isinstance(flow, http.HTTPFlow)
assert flow.websocket is not None
if not flow.websocket.messages:
return searchable.Searchable(
[urwid.Text(("highlight", "No messages."))])
viewmode = self.master.commands.call("console.flowview.mode")
widget_lines = []
for m in flow.websocket.messages:
_, lines, _ = contentviews.get_message_content_view(
viewmode, m, flow)
for line in lines:
if m.from_client:
line.insert(0, self.FROM_CLIENT_MARKER)
else:
line.insert(0, self.TO_CLIENT_MARKER)
widget_lines.append(urwid.Text(line))
if flow.websocket.closed_by_client is not None:
widget_lines.append(
urwid.Text([
(self.FROM_CLIENT_MARKER if flow.websocket.closed_by_client
else self.TO_CLIENT_MARKER),
("alert" if flow.websocket.close_code in (1000, 1001, 1005)
else "error",
f"Connection closed: {flow.websocket.close_code} {flow.websocket.close_reason}"
)
]))
if flow.intercepted:
markup = widget_lines[-1].get_text()[0]
widget_lines[-1].set_text(("intercept", markup))
widget_lines.insert(
0, self._contentview_status_bar(viewmode.capitalize(), viewmode))
return searchable.Searchable(widget_lines)
def _get_content_view(self, viewmode, max_lines, _):
message = self._get_content_view_message
self._get_content_view_message = None
description, lines, error = contentviews.get_message_content_view(
viewmode, message
)
if error:
signals.add_log(error, "error")
# Give hint that you have to tab for the response.
if description == "No content" and isinstance(message, http.HTTPRequest):
description = "No request content (press tab to view response)"
# If the users has a wide terminal, he gets fewer lines; this should not be an issue.
chars_per_line = 80
max_chars = max_lines * chars_per_line
total_chars = 0
text_objects = []
for line in lines:
txt = []
for (style, text) in line:
if total_chars + len(text) > max_chars:
text = text[:max_chars - total_chars]
txt.append((style, text))
total_chars += len(text)
if total_chars == max_chars:
break
# round up to the next line.
total_chars = int(math.ceil(total_chars / chars_per_line) * chars_per_line)
text_objects.append(urwid.Text(txt))
if total_chars == max_chars:
text_objects.append(urwid.Text([
("highlight", "Stopped displaying data after %d lines. Press " % max_lines),
("key", "f"),
("highlight", " to load all data.")
]))
break
return description, text_objects
def flow_to_json_full(flow: mitmproxy.flow.Flow) -> dict:
"""
Remove flow message content and cert to save transmission space.
Args:
flow: The original flow.
"""
f = {
"id": flow.id,
"intercepted": flow.intercepted,
"is_replay": flow.is_replay,
"client_conn": flow.client_conn.get_state(),
"server_conn": flow.server_conn.get_state(),
"type": flow.type,
"modified": flow.modified(),
"marked": flow.marked,
}
# .alpn_proto_negotiated is bytes, we need to decode that.
for conn in "client_conn", "server_conn":
if f[conn]["alpn_proto_negotiated"] is None:
continue
f[conn]["alpn_proto_negotiated"] = \
f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace")
# There are some bytes in here as well, let's skip it until we have them in the UI.
f["client_conn"].pop("tls_extensions", None)
if flow.error:
f["error"] = flow.error.get_state()
if isinstance(flow, http.HTTPFlow):
content_length: Optional[int]
content_hash: Optional[str]
if flow.request:
if flow.request.raw_content:
content_length = len(flow.request.raw_content)
content_hash = hashlib.sha256(
flow.request.raw_content).hexdigest()
req_message = getattr(flow, 'request')
req_description, req_lines, req_error = contentviews.get_message_content_view(
'auto', req_message, flow)
req_content = dict(lines=list(req_lines),
description=req_description)
else:
content_length = None
content_hash = None
req_content = None
f["request"] = {
"method": flow.request.method,
"scheme": flow.request.scheme,
"host": flow.request.host,
"port": flow.request.port,
"path": flow.request.path,
"http_version": flow.request.http_version,
"headers": tuple(flow.request.headers.items(True)),
"contentLength": content_length,
"contentHash": content_hash,
"timestamp_start": flow.request.timestamp_start,
"timestamp_end": flow.request.timestamp_end,
"is_replay": flow.is_replay ==
"request", # TODO: remove, use flow.is_replay instead.
"pretty_host": flow.request.pretty_host,
"req_content": req_content,
}
if flow.response:
if flow.response.raw_content:
content_length = len(flow.response.raw_content)
content_hash = hashlib.sha256(
flow.response.raw_content).hexdigest()
message = getattr(flow, 'response')
description, lines, error = contentviews.get_message_content_view(
'auto', message, flow)
resp_content = dict(lines=list(lines), description=description)
else:
content_length = None
content_hash = None
resp_content = None
f["response"] = {
"http_version": flow.response.http_version,
"status_code": flow.response.status_code,
"reason": flow.response.reason,
"headers": tuple(flow.response.headers.items(True)),
"contentLength": content_length,
"contentHash": content_hash,
"timestamp_start": flow.response.timestamp_start,
"timestamp_end": flow.response.timestamp_end,
"resp_content": resp_content,
"is_replay": flow.is_replay ==
"response", # TODO: remove, use flow.is_replay instead.
}
if flow.response.data.trailers:
f["response"]["trailers"] = tuple(
flow.response.data.trailers.items(True))
f.get("server_conn", {}).pop("cert", None)
f.get("client_conn", {}).pop("mitmcert", None)
return f