Ejemplo n.º 1
0
    def _echo_message(self, message):
        _, lines, error = contentviews.get_message_content_view(
            self.default_contentview,
            message
        )
        if error:
            ctx.log.debug(error)

        if self.flow_detail == 3:
            lines_to_echo = itertools.islice(lines, 70)
        else:
            lines_to_echo = lines

        styles = dict(
            highlight=dict(bold=True),
            offset=dict(fg="blue"),
            header=dict(fg="green", bold=True),
            text=dict(fg="green")
        )

        content = u"\r\n".join(
            u"".join(colorful(line, styles)) for line in lines_to_echo
        )
        if content:
            self.echo("")
            self.echo(content)

        if next(lines, None):
            self.echo("(cut off)", ident=4, dim=True)

        if self.flow_detail >= 2:
            self.echo("")
Ejemplo n.º 2
0
    def _echo_message(
        self,
        message: Union[http.Message, TCPMessage, WebSocketMessage],
        flow: Union[http.HTTPFlow, TCPFlow]
    ):
        _, lines, error = contentviews.get_message_content_view(
            ctx.options.dumper_default_contentview,
            message,
            flow
        )
        if error:
            ctx.log.debug(error)

        if ctx.options.flow_detail == 3:
            lines_to_echo = itertools.islice(lines, 70)
        else:
            lines_to_echo = lines

        content = "\r\n".join(
            "".join(self._colorful(line)) for line in lines_to_echo
        )
        if content:
            self.echo("")
            self.echo(content)

        if next(lines, None):
            self.echo("(cut off)", ident=4, dim=True)

        if ctx.options.flow_detail >= 2:
            self.echo("")
Ejemplo n.º 3
0
def test_get_message_content_view():
    r = tutils.treq()
    desc, lines, err = cv.get_message_content_view("raw", r)
    assert desc == "Raw"

    r.encode("gzip")
    desc, lines, err = cv.get_message_content_view("raw", r)
    assert desc == "[decoded gzip] Raw"

    r.headers["content-encoding"] = "deflate"
    desc, lines, err = cv.get_message_content_view("raw", r)
    assert desc == "[cannot decode] Raw"

    r.content = None
    desc, lines, err = cv.get_message_content_view("raw", r)
    assert list(lines) == [[("error", "content missing")]]
Ejemplo n.º 4
0
def save_flows_content(path: pathlib.Path,
                       flows: typing.Iterable[flow.Flow]) -> None:
    for f in flows:
        for m in ('request', 'response'):
            message = getattr(f, m)
            message_path = path / "flows" / f.id / m
            os.makedirs(str(message_path / "content"), exist_ok=True)

            with open(str(message_path / 'content.data'),
                      'wb') as content_file:
                # don't use raw_content here as this is served with a default content type
                if message:
                    content_file.write(message.content)
                else:
                    content_file.write(b'No content.')

            # content_view
            t = time.time()
            if message:
                description, lines, error = contentviews.get_message_content_view(
                    'Auto', message, f)
            else:
                description, lines = 'No content.', []
            if time.time() - t > 0.1:
                ctx.log(
                    "Slow content view: {} took {}s".format(
                        description.strip(), round(time.time() - t, 1)),
                    "info")
            with open(str(message_path / "content" / "Auto.json"),
                      "w") as content_view_file:
                json.dump(dict(lines=list(lines), description=description),
                          content_view_file)
Ejemplo n.º 5
0
    def _echo_message(self, message: Union[http.Message, TCPMessage,
                                           WebSocketMessage],
                      flow: Union[http.HTTPFlow, TCPFlow, WebSocketFlow]):
        _, lines, error = contentviews.get_message_content_view(
            ctx.options.dumper_default_contentview, message, flow)
        if error:
            ctx.log.debug(error)

        if ctx.options.flow_detail == 3:
            lines_to_echo = itertools.islice(lines, 70)
        else:
            lines_to_echo = lines

        styles = dict(highlight=dict(bold=True),
                      offset=dict(fg="blue"),
                      header=dict(fg="green", bold=True),
                      text=dict(fg="green"))

        content = "\r\n".join("".join(colorful(line, styles))
                              for line in lines_to_echo)
        if content:
            self.echo("")
            self.echo(content)

        if next(lines, None):
            self.echo("(cut off)", ident=4, dim=True)

        if ctx.options.flow_detail >= 2:
            self.echo("")
Ejemplo n.º 6
0
def save_flows_content(path: pathlib.Path, flows: typing.Iterable[flow.Flow]) -> None:
    for f in flows:
        for m in ('request', 'response'):
            message = getattr(f, m)
            message_path = path / "flows" / f.id / m
            os.makedirs(str(message_path / "content"), exist_ok=True)

            with open(str(message_path / 'content.data'), 'wb') as content_file:
                # don't use raw_content here as this is served with a default content type
                if message:
                    content_file.write(message.content)
                else:
                    content_file.write(b'No content.')

            # content_view
            t = time.time()
            if message:
                description, lines, error = contentviews.get_message_content_view(
                    'Auto', message
                )
            else:
                description, lines = 'No content.', []
            if time.time() - t > 0.1:
                ctx.log(
                    "Slow content view: {} took {}s".format(
                        description.strip(),
                        round(time.time() - t, 1)
                    ),
                    "info"
                )
            with open(str(message_path / "content" / "Auto.json"), "w") as content_view_file:
                json.dump(
                    dict(lines=list(lines), description=description),
                    content_view_file
                )
Ejemplo n.º 7
0
    def view_websocket_messages(self):
        flow = self.flow
        assert isinstance(flow, http.HTTPFlow)
        assert flow.websocket is not None

        if not flow.websocket.messages:
            return searchable.Searchable(
                [urwid.Text(("highlight", "No messages."))])

        viewmode = self.master.commands.call("console.flowview.mode")

        widget_lines = []
        for m in flow.websocket.messages:
            _, lines, _ = contentviews.get_message_content_view(
                viewmode, m, flow)

            for line in lines:
                if m.from_client:
                    line.insert(
                        0, ("from_client", f"{common.SYMBOL_FROM_CLIENT} "))
                else:
                    line.insert(0,
                                ("to_client", f"{common.SYMBOL_TO_CLIENT} "))

                widget_lines.append(urwid.Text(line))

        if flow.intercepted:
            markup = widget_lines[-1].get_text()[0]
            widget_lines[-1].set_text(("intercept", markup))

        widget_lines.insert(
            0, self._contentview_status_bar(viewmode.capitalize(), viewmode))

        return searchable.Searchable(widget_lines)
Ejemplo n.º 8
0
    def get(self, flow_id, message, content_view):
        message = getattr(self.flow, message)

        description, lines, error = contentviews.get_message_content_view(
            content_view.replace('_', ' '), message, self.flow)
        #        if error:
        #           add event log

        self.write(dict(lines=list(lines), description=description))
Ejemplo n.º 9
0
    def get(self, flow_id, message, content_view):
        message = getattr(self.flow, message)

        description, lines, error = contentviews.get_message_content_view(
            content_view.replace('_', ' '), message
        )
        #        if error:
        #           add event log

        self.write(dict(
            lines=list(lines),
            description=description
        ))
Ejemplo n.º 10
0
    def _echo_message(self, message):
        if self.flow_detail >= 2 and hasattr(message, "headers"):
            headers = "\r\n".join(
                "{}: {}".format(
                    click.style(
                        strutils.bytes_to_escaped_str(k), fg="blue", bold=True
                    ),
                    click.style(
                        strutils.bytes_to_escaped_str(v), fg="blue"
                    )
                )
                for k, v in message.headers.fields
            )
            self.echo(headers, ident=4)
        if self.flow_detail >= 3:
                _, lines, error = contentviews.get_message_content_view(
                    contentviews.get("Auto"),
                    message
                )
                if error:
                    ctx.log.debug(error)

                styles = dict(
                    highlight=dict(bold=True),
                    offset=dict(fg="blue"),
                    header=dict(fg="green", bold=True),
                    text=dict(fg="green")
                )

                def colorful(line):
                    yield u"    "  # we can already indent here
                    for (style, text) in line:
                        yield click.style(text, **styles.get(style, {}))

                if self.flow_detail == 3:
                    lines_to_echo = itertools.islice(lines, 70)
                else:
                    lines_to_echo = lines

                content = u"\r\n".join(
                    u"".join(colorful(line)) for line in lines_to_echo
                )
                if content:
                    self.echo("")
                    self.echo(content)

                if next(lines, None):
                    self.echo("(cut off)", ident=4, dim=True)

        if self.flow_detail >= 2:
            self.echo("")
Ejemplo n.º 11
0
    def _echo_message(self, message):
        if self.flow_detail >= 2 and hasattr(message, "headers"):
            headers = "\r\n".join(
                "{}: {}".format(
                    click.style(
                        strutils.bytes_to_escaped_str(k), fg="blue", bold=True
                    ),
                    click.style(
                        strutils.bytes_to_escaped_str(v), fg="blue"
                    )
                )
                for k, v in message.headers.fields
            )
            self.echo(headers, ident=4)
        if self.flow_detail >= 3:
                _, lines, error = contentviews.get_message_content_view(
                    self.default_contentview,
                    message
                )
                if error:
                    ctx.log.debug(error)

                styles = dict(
                    highlight=dict(bold=True),
                    offset=dict(fg="blue"),
                    header=dict(fg="green", bold=True),
                    text=dict(fg="green")
                )

                def colorful(line):
                    yield u"    "  # we can already indent here
                    for (style, text) in line:
                        yield click.style(text, **styles.get(style, {}))

                if self.flow_detail == 3:
                    lines_to_echo = itertools.islice(lines, 70)
                else:
                    lines_to_echo = lines

                content = u"\r\n".join(
                    u"".join(colorful(line)) for line in lines_to_echo
                )
                if content:
                    self.echo("")
                    self.echo(content)

                if next(lines, None):
                    self.echo("(cut off)", ident=4, dim=True)

        if self.flow_detail >= 2:
            self.echo("")
Ejemplo n.º 12
0
    def _get_content_view(self, viewmode, max_lines, flowtype, _):
        message = self._get_content_view_message
        self._get_content_view_message = None
        _message = message
        if flowtype is FlowType.Request:
            message = message.request
        else:
            message = message.response
        description, lines, error = contentviews.get_message_content_view(
            viewmode, _message, flowtype)
        if error:
            signals.add_log(error, "error")
        # Give hint that you have to tab for the response.
        if description == "No content" and isinstance(message,
                                                      http.HTTPRequest):
            description = "No request content (press tab to view response)"

        # If the users has a wide terminal, he gets fewer lines; this should not be an issue.
        chars_per_line = 80
        max_chars = max_lines * chars_per_line
        total_chars = 0
        text_objects = []
        for line in lines:
            txt = []
            for (style, text) in line:
                if total_chars + len(text) > max_chars:
                    text = text[:max_chars - total_chars]
                txt.append((style, text))
                total_chars += len(text)
                if total_chars == max_chars:
                    break

            # round up to the next line.
            total_chars = int(
                math.ceil(total_chars / chars_per_line) * chars_per_line)

            text_objects.append(urwid.Text(txt))
            if total_chars == max_chars:
                text_objects.append(
                    urwid.Text([
                        ("highlight",
                         "Stopped displaying data after %d lines. Press " %
                         max_lines), ("key", "f"),
                        ("highlight", " to load all data.")
                    ]))
                break

        return description, text_objects
Ejemplo n.º 13
0
    def message_to_json(
        self,
        viewname: str,
        message: Union[http.Message, TCPMessage, WebSocketMessage],
        flow: Union[HTTPFlow, TCPFlow],
        max_lines: Optional[int] = None
    ):
        description, lines, error = contentviews.get_message_content_view(viewname, message, flow)
        if error:
            self.master.log.error(error)
        if max_lines:
            lines = islice(lines, max_lines)

        return dict(
            lines=list(lines),
            description=description,
        )
Ejemplo n.º 14
0
    def view_websocket_messages(self):
        flow = self.flow
        assert isinstance(flow, http.HTTPFlow)
        assert flow.websocket is not None

        if not flow.websocket.messages:
            return searchable.Searchable(
                [urwid.Text(("highlight", "No messages."))])

        viewmode = self.master.commands.call("console.flowview.mode")

        widget_lines = []
        for m in flow.websocket.messages:
            _, lines, _ = contentviews.get_message_content_view(
                viewmode, m, flow)

            for line in lines:
                if m.from_client:
                    line.insert(0, self.FROM_CLIENT_MARKER)
                else:
                    line.insert(0, self.TO_CLIENT_MARKER)

                widget_lines.append(urwid.Text(line))

        if flow.websocket.closed_by_client is not None:
            widget_lines.append(
                urwid.Text([
                    (self.FROM_CLIENT_MARKER if flow.websocket.closed_by_client
                     else self.TO_CLIENT_MARKER),
                    ("alert" if flow.websocket.close_code in (1000, 1001, 1005)
                     else "error",
                     f"Connection closed: {flow.websocket.close_code} {flow.websocket.close_reason}"
                     )
                ]))

        if flow.intercepted:
            markup = widget_lines[-1].get_text()[0]
            widget_lines[-1].set_text(("intercept", markup))

        widget_lines.insert(
            0, self._contentview_status_bar(viewmode.capitalize(), viewmode))

        return searchable.Searchable(widget_lines)
Ejemplo n.º 15
0
    def _get_content_view(self, viewmode, max_lines, _):
        message = self._get_content_view_message
        self._get_content_view_message = None
        description, lines, error = contentviews.get_message_content_view(
            viewmode, message
        )
        if error:
            signals.add_log(error, "error")
        # Give hint that you have to tab for the response.
        if description == "No content" and isinstance(message, http.HTTPRequest):
            description = "No request content (press tab to view response)"

        # If the users has a wide terminal, he gets fewer lines; this should not be an issue.
        chars_per_line = 80
        max_chars = max_lines * chars_per_line
        total_chars = 0
        text_objects = []
        for line in lines:
            txt = []
            for (style, text) in line:
                if total_chars + len(text) > max_chars:
                    text = text[:max_chars - total_chars]
                txt.append((style, text))
                total_chars += len(text)
                if total_chars == max_chars:
                    break

            # round up to the next line.
            total_chars = int(math.ceil(total_chars / chars_per_line) * chars_per_line)

            text_objects.append(urwid.Text(txt))
            if total_chars == max_chars:
                text_objects.append(urwid.Text([
                    ("highlight", "Stopped displaying data after %d lines. Press " % max_lines),
                    ("key", "f"),
                    ("highlight", " to load all data.")
                ]))
                break

        return description, text_objects
Ejemplo n.º 16
0
def flow_to_json_full(flow: mitmproxy.flow.Flow) -> dict:
    """
    Remove flow message content and cert to save transmission space.

    Args:
        flow: The original flow.
    """
    f = {
        "id": flow.id,
        "intercepted": flow.intercepted,
        "is_replay": flow.is_replay,
        "client_conn": flow.client_conn.get_state(),
        "server_conn": flow.server_conn.get_state(),
        "type": flow.type,
        "modified": flow.modified(),
        "marked": flow.marked,
    }
    # .alpn_proto_negotiated is bytes, we need to decode that.
    for conn in "client_conn", "server_conn":
        if f[conn]["alpn_proto_negotiated"] is None:
            continue
        f[conn]["alpn_proto_negotiated"] = \
            f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace")
    # There are some bytes in here as well, let's skip it until we have them in the UI.
    f["client_conn"].pop("tls_extensions", None)
    if flow.error:
        f["error"] = flow.error.get_state()

    if isinstance(flow, http.HTTPFlow):
        content_length: Optional[int]
        content_hash: Optional[str]
        if flow.request:
            if flow.request.raw_content:
                content_length = len(flow.request.raw_content)
                content_hash = hashlib.sha256(
                    flow.request.raw_content).hexdigest()
                req_message = getattr(flow, 'request')
                req_description, req_lines, req_error = contentviews.get_message_content_view(
                    'auto', req_message, flow)
                req_content = dict(lines=list(req_lines),
                                   description=req_description)
            else:
                content_length = None
                content_hash = None
                req_content = None
            f["request"] = {
                "method": flow.request.method,
                "scheme": flow.request.scheme,
                "host": flow.request.host,
                "port": flow.request.port,
                "path": flow.request.path,
                "http_version": flow.request.http_version,
                "headers": tuple(flow.request.headers.items(True)),
                "contentLength": content_length,
                "contentHash": content_hash,
                "timestamp_start": flow.request.timestamp_start,
                "timestamp_end": flow.request.timestamp_end,
                "is_replay": flow.is_replay ==
                "request",  # TODO: remove, use flow.is_replay instead.
                "pretty_host": flow.request.pretty_host,
                "req_content": req_content,
            }
        if flow.response:
            if flow.response.raw_content:
                content_length = len(flow.response.raw_content)
                content_hash = hashlib.sha256(
                    flow.response.raw_content).hexdigest()
                message = getattr(flow, 'response')
                description, lines, error = contentviews.get_message_content_view(
                    'auto', message, flow)
                resp_content = dict(lines=list(lines), description=description)
            else:
                content_length = None
                content_hash = None
                resp_content = None
            f["response"] = {
                "http_version": flow.response.http_version,
                "status_code": flow.response.status_code,
                "reason": flow.response.reason,
                "headers": tuple(flow.response.headers.items(True)),
                "contentLength": content_length,
                "contentHash": content_hash,
                "timestamp_start": flow.response.timestamp_start,
                "timestamp_end": flow.response.timestamp_end,
                "resp_content": resp_content,
                "is_replay": flow.is_replay ==
                "response",  # TODO: remove, use flow.is_replay instead.
            }
            if flow.response.data.trailers:
                f["response"]["trailers"] = tuple(
                    flow.response.data.trailers.items(True))

    f.get("server_conn", {}).pop("cert", None)
    f.get("client_conn", {}).pop("mitmcert", None)

    return f