def _echo_message(self, message): _, lines, error = contentviews.get_message_content_view( self.default_contentview, message ) if error: ctx.log.debug(error) if self.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines styles = dict( highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green") ) content = u"\r\n".join( u"".join(colorful(line, styles)) for line in lines_to_echo ) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if self.flow_detail >= 2: self.echo("")
def _echo_message( self, message: Union[http.Message, TCPMessage, WebSocketMessage], flow: Union[http.HTTPFlow, TCPFlow] ): _, lines, error = contentviews.get_message_content_view( ctx.options.dumper_default_contentview, message, flow ) if error: ctx.log.debug(error) if ctx.options.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines content = "\r\n".join( "".join(self._colorful(line)) for line in lines_to_echo ) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if ctx.options.flow_detail >= 2: self.echo("")
def test_get_message_content_view(): r = tutils.treq() desc, lines, err = cv.get_message_content_view("raw", r) assert desc == "Raw" r.encode("gzip") desc, lines, err = cv.get_message_content_view("raw", r) assert desc == "[decoded gzip] Raw" r.headers["content-encoding"] = "deflate" desc, lines, err = cv.get_message_content_view("raw", r) assert desc == "[cannot decode] Raw" r.content = None desc, lines, err = cv.get_message_content_view("raw", r) assert list(lines) == [[("error", "content missing")]]
def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: for f in flows: for m in ('request', 'response'): message = getattr(f, m) message_path = path / "flows" / f.id / m os.makedirs(str(message_path / "content"), exist_ok=True) with open(str(message_path / 'content.data'), 'wb') as content_file: # don't use raw_content here as this is served with a default content type if message: content_file.write(message.content) else: content_file.write(b'No content.') # content_view t = time.time() if message: description, lines, error = contentviews.get_message_content_view( 'Auto', message, f) else: description, lines = 'No content.', [] if time.time() - t > 0.1: ctx.log( "Slow content view: {} took {}s".format( description.strip(), round(time.time() - t, 1)), "info") with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file: json.dump(dict(lines=list(lines), description=description), content_view_file)
def _echo_message(self, message: Union[http.Message, TCPMessage, WebSocketMessage], flow: Union[http.HTTPFlow, TCPFlow, WebSocketFlow]): _, lines, error = contentviews.get_message_content_view( ctx.options.dumper_default_contentview, message, flow) if error: ctx.log.debug(error) if ctx.options.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines styles = dict(highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green")) content = "\r\n".join("".join(colorful(line, styles)) for line in lines_to_echo) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if ctx.options.flow_detail >= 2: self.echo("")
def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None: for f in flows: for m in ('request', 'response'): message = getattr(f, m) message_path = path / "flows" / f.id / m os.makedirs(str(message_path / "content"), exist_ok=True) with open(str(message_path / 'content.data'), 'wb') as content_file: # don't use raw_content here as this is served with a default content type if message: content_file.write(message.content) else: content_file.write(b'No content.') # content_view t = time.time() if message: description, lines, error = contentviews.get_message_content_view( 'Auto', message ) else: description, lines = 'No content.', [] if time.time() - t > 0.1: ctx.log( "Slow content view: {} took {}s".format( description.strip(), round(time.time() - t, 1) ), "info" ) with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file: json.dump( dict(lines=list(lines), description=description), content_view_file )
def view_websocket_messages(self): flow = self.flow assert isinstance(flow, http.HTTPFlow) assert flow.websocket is not None if not flow.websocket.messages: return searchable.Searchable( [urwid.Text(("highlight", "No messages."))]) viewmode = self.master.commands.call("console.flowview.mode") widget_lines = [] for m in flow.websocket.messages: _, lines, _ = contentviews.get_message_content_view( viewmode, m, flow) for line in lines: if m.from_client: line.insert( 0, ("from_client", f"{common.SYMBOL_FROM_CLIENT} ")) else: line.insert(0, ("to_client", f"{common.SYMBOL_TO_CLIENT} ")) widget_lines.append(urwid.Text(line)) if flow.intercepted: markup = widget_lines[-1].get_text()[0] widget_lines[-1].set_text(("intercept", markup)) widget_lines.insert( 0, self._contentview_status_bar(viewmode.capitalize(), viewmode)) return searchable.Searchable(widget_lines)
def get(self, flow_id, message, content_view): message = getattr(self.flow, message) description, lines, error = contentviews.get_message_content_view( content_view.replace('_', ' '), message, self.flow) # if error: # add event log self.write(dict(lines=list(lines), description=description))
def get(self, flow_id, message, content_view): message = getattr(self.flow, message) description, lines, error = contentviews.get_message_content_view( content_view.replace('_', ' '), message ) # if error: # add event log self.write(dict( lines=list(lines), description=description ))
def _echo_message(self, message): if self.flow_detail >= 2 and hasattr(message, "headers"): headers = "\r\n".join( "{}: {}".format( click.style( strutils.bytes_to_escaped_str(k), fg="blue", bold=True ), click.style( strutils.bytes_to_escaped_str(v), fg="blue" ) ) for k, v in message.headers.fields ) self.echo(headers, ident=4) if self.flow_detail >= 3: _, lines, error = contentviews.get_message_content_view( contentviews.get("Auto"), message ) if error: ctx.log.debug(error) styles = dict( highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green") ) def colorful(line): yield u" " # we can already indent here for (style, text) in line: yield click.style(text, **styles.get(style, {})) if self.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines content = u"\r\n".join( u"".join(colorful(line)) for line in lines_to_echo ) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if self.flow_detail >= 2: self.echo("")
def _echo_message(self, message): if self.flow_detail >= 2 and hasattr(message, "headers"): headers = "\r\n".join( "{}: {}".format( click.style( strutils.bytes_to_escaped_str(k), fg="blue", bold=True ), click.style( strutils.bytes_to_escaped_str(v), fg="blue" ) ) for k, v in message.headers.fields ) self.echo(headers, ident=4) if self.flow_detail >= 3: _, lines, error = contentviews.get_message_content_view( self.default_contentview, message ) if error: ctx.log.debug(error) styles = dict( highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green") ) def colorful(line): yield u" " # we can already indent here for (style, text) in line: yield click.style(text, **styles.get(style, {})) if self.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines content = u"\r\n".join( u"".join(colorful(line)) for line in lines_to_echo ) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if self.flow_detail >= 2: self.echo("")
def _get_content_view(self, viewmode, max_lines, flowtype, _): message = self._get_content_view_message self._get_content_view_message = None _message = message if flowtype is FlowType.Request: message = message.request else: message = message.response description, lines, error = contentviews.get_message_content_view( viewmode, _message, flowtype) if error: signals.add_log(error, "error") # Give hint that you have to tab for the response. if description == "No content" and isinstance(message, http.HTTPRequest): description = "No request content (press tab to view response)" # If the users has a wide terminal, he gets fewer lines; this should not be an issue. chars_per_line = 80 max_chars = max_lines * chars_per_line total_chars = 0 text_objects = [] for line in lines: txt = [] for (style, text) in line: if total_chars + len(text) > max_chars: text = text[:max_chars - total_chars] txt.append((style, text)) total_chars += len(text) if total_chars == max_chars: break # round up to the next line. total_chars = int( math.ceil(total_chars / chars_per_line) * chars_per_line) text_objects.append(urwid.Text(txt)) if total_chars == max_chars: text_objects.append( urwid.Text([ ("highlight", "Stopped displaying data after %d lines. Press " % max_lines), ("key", "f"), ("highlight", " to load all data.") ])) break return description, text_objects
def message_to_json( self, viewname: str, message: Union[http.Message, TCPMessage, WebSocketMessage], flow: Union[HTTPFlow, TCPFlow], max_lines: Optional[int] = None ): description, lines, error = contentviews.get_message_content_view(viewname, message, flow) if error: self.master.log.error(error) if max_lines: lines = islice(lines, max_lines) return dict( lines=list(lines), description=description, )
def view_websocket_messages(self): flow = self.flow assert isinstance(flow, http.HTTPFlow) assert flow.websocket is not None if not flow.websocket.messages: return searchable.Searchable( [urwid.Text(("highlight", "No messages."))]) viewmode = self.master.commands.call("console.flowview.mode") widget_lines = [] for m in flow.websocket.messages: _, lines, _ = contentviews.get_message_content_view( viewmode, m, flow) for line in lines: if m.from_client: line.insert(0, self.FROM_CLIENT_MARKER) else: line.insert(0, self.TO_CLIENT_MARKER) widget_lines.append(urwid.Text(line)) if flow.websocket.closed_by_client is not None: widget_lines.append( urwid.Text([ (self.FROM_CLIENT_MARKER if flow.websocket.closed_by_client else self.TO_CLIENT_MARKER), ("alert" if flow.websocket.close_code in (1000, 1001, 1005) else "error", f"Connection closed: {flow.websocket.close_code} {flow.websocket.close_reason}" ) ])) if flow.intercepted: markup = widget_lines[-1].get_text()[0] widget_lines[-1].set_text(("intercept", markup)) widget_lines.insert( 0, self._contentview_status_bar(viewmode.capitalize(), viewmode)) return searchable.Searchable(widget_lines)
def _get_content_view(self, viewmode, max_lines, _): message = self._get_content_view_message self._get_content_view_message = None description, lines, error = contentviews.get_message_content_view( viewmode, message ) if error: signals.add_log(error, "error") # Give hint that you have to tab for the response. if description == "No content" and isinstance(message, http.HTTPRequest): description = "No request content (press tab to view response)" # If the users has a wide terminal, he gets fewer lines; this should not be an issue. chars_per_line = 80 max_chars = max_lines * chars_per_line total_chars = 0 text_objects = [] for line in lines: txt = [] for (style, text) in line: if total_chars + len(text) > max_chars: text = text[:max_chars - total_chars] txt.append((style, text)) total_chars += len(text) if total_chars == max_chars: break # round up to the next line. total_chars = int(math.ceil(total_chars / chars_per_line) * chars_per_line) text_objects.append(urwid.Text(txt)) if total_chars == max_chars: text_objects.append(urwid.Text([ ("highlight", "Stopped displaying data after %d lines. Press " % max_lines), ("key", "f"), ("highlight", " to load all data.") ])) break return description, text_objects
def flow_to_json_full(flow: mitmproxy.flow.Flow) -> dict: """ Remove flow message content and cert to save transmission space. Args: flow: The original flow. """ f = { "id": flow.id, "intercepted": flow.intercepted, "is_replay": flow.is_replay, "client_conn": flow.client_conn.get_state(), "server_conn": flow.server_conn.get_state(), "type": flow.type, "modified": flow.modified(), "marked": flow.marked, } # .alpn_proto_negotiated is bytes, we need to decode that. for conn in "client_conn", "server_conn": if f[conn]["alpn_proto_negotiated"] is None: continue f[conn]["alpn_proto_negotiated"] = \ f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace") # There are some bytes in here as well, let's skip it until we have them in the UI. f["client_conn"].pop("tls_extensions", None) if flow.error: f["error"] = flow.error.get_state() if isinstance(flow, http.HTTPFlow): content_length: Optional[int] content_hash: Optional[str] if flow.request: if flow.request.raw_content: content_length = len(flow.request.raw_content) content_hash = hashlib.sha256( flow.request.raw_content).hexdigest() req_message = getattr(flow, 'request') req_description, req_lines, req_error = contentviews.get_message_content_view( 'auto', req_message, flow) req_content = dict(lines=list(req_lines), description=req_description) else: content_length = None content_hash = None req_content = None f["request"] = { "method": flow.request.method, "scheme": flow.request.scheme, "host": flow.request.host, "port": flow.request.port, "path": flow.request.path, "http_version": flow.request.http_version, "headers": tuple(flow.request.headers.items(True)), "contentLength": content_length, "contentHash": content_hash, "timestamp_start": flow.request.timestamp_start, "timestamp_end": flow.request.timestamp_end, "is_replay": flow.is_replay == "request", # TODO: remove, use flow.is_replay instead. "pretty_host": flow.request.pretty_host, "req_content": req_content, } if flow.response: if flow.response.raw_content: content_length = len(flow.response.raw_content) content_hash = hashlib.sha256( flow.response.raw_content).hexdigest() message = getattr(flow, 'response') description, lines, error = contentviews.get_message_content_view( 'auto', message, flow) resp_content = dict(lines=list(lines), description=description) else: content_length = None content_hash = None resp_content = None f["response"] = { "http_version": flow.response.http_version, "status_code": flow.response.status_code, "reason": flow.response.reason, "headers": tuple(flow.response.headers.items(True)), "contentLength": content_length, "contentHash": content_hash, "timestamp_start": flow.response.timestamp_start, "timestamp_end": flow.response.timestamp_end, "resp_content": resp_content, "is_replay": flow.is_replay == "response", # TODO: remove, use flow.is_replay instead. } if flow.response.data.trailers: f["response"]["trailers"] = tuple( flow.response.data.trailers.items(True)) f.get("server_conn", {}).pop("cert", None) f.get("client_conn", {}).pop("mitmcert", None) return f