def _echo_request_line(self, flow): if flow.request.stickycookie: stickycookie = click.style("[stickycookie] ", fg="yellow", bold=True) else: stickycookie = "" if flow.client_conn: client = click.style(strutils.bytes_to_escaped_str(flow.client_conn.address.host), bold=True) else: client = click.style("[replay]", fg="yellow", bold=True) method = flow.request.method method_color = dict( GET="green", DELETE="red" ).get(method.upper(), "magenta") method = click.style(strutils.bytes_to_escaped_str(method), fg=method_color, bold=True) if self.showhost: url = flow.request.pretty_url else: url = flow.request.url url = click.style(strutils.bytes_to_escaped_str(url), bold=True) httpversion = "" if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): httpversion = " " + flow.request.http_version # We hide "normal" HTTP 1. line = "{stickycookie}{client} {method} {url}{httpversion}".format( stickycookie=stickycookie, client=client, method=method, url=url, httpversion=httpversion ) self.echo(line)
def _echo_message(self, message): if self.o.flow_detail >= 2: headers = "\r\n".join( "{}: {}".format( click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True), click.style(strutils.bytes_to_escaped_str(v), fg="blue")) for k, v in message.headers.fields ) self.echo(headers, indent=4) if self.o.flow_detail >= 3: if message.content is None: self.echo("(content missing)", indent=4) elif message.content: self.echo("") try: type, lines = contentviews.get_content_view( contentviews.get("Auto"), message.content, headers=message.headers ) except exceptions.ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() self.add_event(s, "debug") type, lines = contentviews.get_content_view( contentviews.get("Raw"), message.content, headers=message.headers ) styles = dict( highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green") ) def colorful(line): yield u" " # we can already indent here for (style, text) in line: yield click.style(text, **styles.get(style, {})) if self.o.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines lines_to_echo = list(lines_to_echo) content = u"\r\n".join( u"".join(colorful(line)) for line in lines_to_echo ) self.echo(content) if next(lines, None): self.echo("(cut off)", indent=4, dim=True) if self.o.flow_detail >= 2: self.echo("")
def _echo_message(self, message): if self.o.flow_detail >= 2: headers = "\r\n".join("{}: {}".format( click.style( strutils.bytes_to_escaped_str(k), fg="blue", bold=True), click.style(strutils.bytes_to_escaped_str(v), fg="blue")) for k, v in message.headers.fields) self.echo(headers, indent=4) if self.o.flow_detail >= 3: if message.content is None: self.echo("(content missing)", indent=4) elif message.content: self.echo("") try: type, lines = contentviews.get_content_view( contentviews.get("Auto"), message.content, headers=message.headers) except exceptions.ContentViewException: s = "Content viewer failed: \n" + traceback.format_exc() self.add_event(s, "debug") type, lines = contentviews.get_content_view( contentviews.get("Raw"), message.content, headers=message.headers) styles = dict(highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green")) def colorful(line): yield u" " # we can already indent here for (style, text) in line: yield click.style(text, **styles.get(style, {})) if self.o.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines lines_to_echo = list(lines_to_echo) content = u"\r\n".join(u"".join(colorful(line)) for line in lines_to_echo) self.echo(content) if next(lines, None): self.echo("(cut off)", indent=4, dim=True) if self.o.flow_detail >= 2: self.echo("")
def _echo_message(self, message): if self.flow_detail >= 2 and hasattr(message, "headers"): headers = "\r\n".join( "{}: {}".format( click.style( strutils.bytes_to_escaped_str(k), fg="blue", bold=True ), click.style( strutils.bytes_to_escaped_str(v), fg="blue" ) ) for k, v in message.headers.fields ) self.echo(headers, ident=4) if self.flow_detail >= 3: _, lines, error = contentviews.get_message_content_view( contentviews.get("Auto"), message ) if error: ctx.log.debug(error) styles = dict( highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green") ) def colorful(line): yield u" " # we can already indent here for (style, text) in line: yield click.style(text, **styles.get(style, {})) if self.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines content = u"\r\n".join( u"".join(colorful(line)) for line in lines_to_echo ) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if self.flow_detail >= 2: self.echo("")
def _echo_response_line(self, flow): if flow.response.is_replay: replay = click.style("[replay] ", fg="yellow", bold=True) else: replay = "" code = flow.response.status_code code_color = None if 200 <= code < 300: code_color = "green" elif 300 <= code < 400: code_color = "magenta" elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) reason = click.style(strutils.bytes_to_escaped_str(flow.response.reason), fg=code_color, bold=True) if flow.response.content is None: size = "(content missing)" else: size = human.pretty_size(len(flow.response.content)) size = click.style(size, bold=True) arrows = click.style("<<", bold=True) line = "{replay} {arrows} {code} {reason} {size}".format( replay=replay, arrows=arrows, code=code, reason=reason, size=size ) self.echo(line)
def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"foo") == "foo" assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" assert strutils.bytes_to_escaped_str(b"'") == r"\'" assert strutils.bytes_to_escaped_str(b'"') == r'"' with tutils.raises(ValueError): strutils.bytes_to_escaped_str(u"such unicode")
def log(self, settings): """ A dictionary that should be logged if this message is served. """ ret = {} for i in self.logattrs: v = getattr(self, i) # Careful not to log any VALUE specs without sanitizing them first. # We truncate at 1k. if hasattr(v, "values"): v = [x[:LOG_TRUNCATE] for x in v.values(settings)] v = strutils.bytes_to_escaped_str(b"".join(v)) elif hasattr(v, "__len__"): v = v[:LOG_TRUNCATE] v = strutils.bytes_to_escaped_str(v) ret[i] = v ret["spec"] = self.spec() return ret
def tcp_message(self, f): super(ConsoleMaster, self).tcp_message(f) message = f.messages[-1] direction = "->" if message.from_client else "<-" self.add_log("{client} {direction} tcp {direction} {server}".format( client=repr(f.client_conn.address), server=repr(f.server_conn.address), direction=direction, ), "info") self.add_log(strutils.bytes_to_escaped_str(message.content), "debug")
def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"foo") == "foo" assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" assert strutils.bytes_to_escaped_str(b"'") == r"\'" assert strutils.bytes_to_escaped_str(b'"') == r'"'
def _echo_message(self, message): if self.flow_detail >= 2 and hasattr(message, "headers"): headers = "\r\n".join("{}: {}".format( click.style( strutils.bytes_to_escaped_str(k), fg="blue", bold=True), click.style(strutils.bytes_to_escaped_str(v), fg="blue")) for k, v in message.headers.fields) self.echo(headers, ident=4) if self.flow_detail >= 3: _, lines, error = contentviews.get_message_content_view( contentviews.get("Auto"), message) if error: ctx.log.debug(error) styles = dict(highlight=dict(bold=True), offset=dict(fg="blue"), header=dict(fg="green", bold=True), text=dict(fg="green")) def colorful(line): yield u" " # we can already indent here for (style, text) in line: yield click.style(text, **styles.get(style, {})) if self.flow_detail == 3: lines_to_echo = itertools.islice(lines, 70) else: lines_to_echo = lines content = u"\r\n".join(u"".join(colorful(line)) for line in lines_to_echo) if content: self.echo("") self.echo(content) if next(lines, None): self.echo("(cut off)", ident=4, dim=True) if self.flow_detail >= 2: self.echo("")
def tcp_message(tcp_msg): modified_msg = tcp_msg.message.replace("foo", "bar") is_modified = False if modified_msg == tcp_msg.message else True tcp_msg.message = modified_msg print("[tcp_message{}] from {} {} to {} {}:\r\n{}".format( " (modified)" if is_modified else "", "client" if tcp_msg.sender == tcp_msg.client_conn else "server", tcp_msg.sender.address, "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message)))
def _preview(is_request): if is_request: template = "request_preview.html" else: template = "response_preview.html" spec = request.args["spec"] args = dict( spec=spec, section="main", syntaxerror=None, error=None, ) if not spec.strip(): args["error"] = "Can't parse an empty spec." return render(template, False, **args) try: if is_request: r = language.parse_pathoc(spec).next() else: r = language.parse_pathod(spec).next() except language.ParseException as v: args["syntaxerror"] = str(v) args["marked"] = v.marked() return render(template, False, **args) s = io.BytesIO() settings = copy.copy(app.config["pathod"].settings) settings.request_host = EXAMPLE_HOST settings.websocket_key = EXAMPLE_WEBSOCKET_KEY safe = r.preview_safe() err, safe = app.config["pathod"].check_policy( safe, settings ) if err: args["error"] = err return render(template, False, **args) if is_request: settings.request_host = EXAMPLE_HOST language.serve(safe, s, settings) else: settings.websocket_key = EXAMPLE_WEBSOCKET_KEY language.serve(safe, s, settings) args["output"] = strutils.bytes_to_escaped_str(s.getvalue()) return render(template, False, **args)
def _echo_request_line(self, flow): if flow.request.stickycookie: stickycookie = click.style("[stickycookie] ", fg="yellow", bold=True) else: stickycookie = "" if flow.client_conn: client = click.style(strutils.bytes_to_escaped_str( flow.client_conn.address.host), bold=True) else: client = click.style("[replay]", fg="yellow", bold=True) method = flow.request.method method_color = dict(GET="green", DELETE="red").get(method.upper(), "magenta") method = click.style(strutils.bytes_to_escaped_str(method), fg=method_color, bold=True) if self.showhost: url = flow.request.pretty_url else: url = flow.request.url url = click.style(strutils.bytes_to_escaped_str(url), bold=True) httpversion = "" if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): httpversion = " " + flow.request.http_version # We hide "normal" HTTP 1. line = "{stickycookie}{client} {method} {url}{httpversion}".format( stickycookie=stickycookie, client=client, method=method, url=url, httpversion=httpversion) self.echo(line)
def tcp_message(tcp_msg): modified_msg = tcp_msg.message.replace("foo", "bar") is_modified = False if modified_msg == tcp_msg.message else True tcp_msg.message = modified_msg print( "[tcp_message{}] from {} {} to {} {}:\r\n{}".format( " (modified)" if is_modified else "", "client" if tcp_msg.sender == tcp_msg.client_conn else "server", tcp_msg.sender.address, "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message)) )
def http(self, r): """ Performs a single request. r: A language.http.Request object, or a string representing one request. Returns Response if we have a non-ignored response. May raise a exceptions.NetlibException """ logger = log.ConnectionLogger( self.fp, self.hexdump, False, self.rfile if self.showresp else None, self.wfile if self.showreq else None, ) with logger.ctx() as lg: lg(">> %s" % r) resp, req = None, None try: req = language.serve(r, self.wfile, self.settings) self.wfile.flush() resp = self.protocol.read_response( self.rfile, treq(method=req["method"].encode())) resp.sslinfo = self.sslinfo except exceptions.HttpException as v: lg("Invalid server response: %s" % v) raise except exceptions.TcpTimeout: if self.ignoretimeout: lg("Timeout (ignored)") return None lg("Timeout") raise finally: if resp: lg("<< %s %s: %s bytes" % (resp.status_code, strutils.bytes_to_escaped_str( resp.reason), len(resp.content))) if resp.status_code in self.ignorecodes: lg.suppress() return resp
def _handle_frame(self, frame, source_conn, other_conn, is_server): sender = "server" if is_server else "client" self.log("WebSockets Frame received from {}".format(sender), "debug", [repr(frame)]) if frame.header.opcode & 0x8 == 0: self.log( "{direction} websocket {direction} {server}".format( server=repr(self.server_conn.address), direction="<-" if is_server else "->", ), "info", strutils.bytes_to_escaped_str(frame.payload, keep_spacing=True).splitlines()) # forward the data frame to the other side other_conn.send(bytes(frame)) elif frame.header.opcode in (websockets.OPCODE.PING, websockets.OPCODE.PONG): # just forward the ping/pong to the other side other_conn.send(bytes(frame)) elif frame.header.opcode == websockets.OPCODE.CLOSE: code = '(status code missing)' msg = None reason = '(message missing)' if len(frame.payload) >= 2: code, = struct.unpack('!H', frame.payload[:2]) msg = websockets.CLOSE_REASON.get_name( code, default='unknown status code') if len(frame.payload) > 2: reason = frame.payload[2:] self.log( "WebSockets connection closed by {}: {} {}, {}".format( sender, code, msg, reason), "info") other_conn.send(bytes(frame)) # close the connection return False else: self.log( "Unknown WebSockets frame received from {}".format(sender), "info", [repr(frame)]) # unknown frame - just forward it other_conn.send(bytes(frame)) # continue the connection return True
def http(self, r): """ Performs a single request. r: A language.http.Request object, or a string representing one request. Returns Response if we have a non-ignored response. May raise a exceptions.NetlibException """ logger = log.ConnectionLogger( self.fp, self.hexdump, False, self.rfile if self.showresp else None, self.wfile if self.showreq else None, ) with logger.ctx() as lg: lg(">> %s" % r) resp, req = None, None try: req = language.serve(r, self.wfile, self.settings) self.wfile.flush() resp = self.protocol.read_response(self.rfile, treq(method=req["method"].encode())) resp.sslinfo = self.sslinfo except exceptions.HttpException as v: lg("Invalid server response: %s" % v) raise except exceptions.TcpTimeout: if self.ignoretimeout: lg("Timeout (ignored)") return None lg("Timeout") raise finally: if resp: lg("<< %s %s: %s bytes" % ( resp.status_code, strutils.bytes_to_escaped_str(resp.reason), len(resp.content) )) if resp.status_code in self.ignorecodes: lg.suppress() return resp
def _handle_frame(self, frame, source_conn, other_conn, is_server): sender = "server" if is_server else "client" self.log( "WebSockets Frame received from {}".format(sender), "debug", [repr(frame)] ) if frame.header.opcode & 0x8 == 0: self.log( "{direction} websocket {direction} {server}".format( server=repr(self.server_conn.address), direction="<-" if is_server else "->", ), "info", strutils.bytes_to_escaped_str(frame.payload, keep_spacing=True).splitlines() ) # forward the data frame to the other side other_conn.send(bytes(frame)) elif frame.header.opcode in (websockets.OPCODE.PING, websockets.OPCODE.PONG): # just forward the ping/pong to the other side other_conn.send(bytes(frame)) elif frame.header.opcode == websockets.OPCODE.CLOSE: code = '(status code missing)' msg = None reason = '(message missing)' if len(frame.payload) >= 2: code, = struct.unpack('!H', frame.payload[:2]) msg = websockets.CLOSE_REASON.get_name(code, default='unknown status code') if len(frame.payload) > 2: reason = frame.payload[2:] self.log("WebSockets connection closed by {}: {} {}, {}".format(sender, code, msg, reason), "info") other_conn.send(bytes(frame)) # close the connection return False else: self.log("Unknown WebSockets frame received from {}".format(sender), "info", [repr(frame)]) # unknown frame - just forward it other_conn.send(bytes(frame)) # continue the connection return True
def _echo_response_line(self, flow): if flow.response.is_replay: replay = click.style("[replay] ", fg="yellow", bold=True) else: replay = "" code = flow.response.status_code code_color = None if 200 <= code < 300: code_color = "green" elif 300 <= code < 400: code_color = "magenta" elif 400 <= code < 600: code_color = "red" code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) reason = click.style(strutils.bytes_to_escaped_str( flow.response.reason), fg=code_color, bold=True) if flow.response.content is None: size = "(content missing)" else: size = human.pretty_size(len(flow.response.content)) size = click.style(size, bold=True) arrows = click.style("<<", bold=True) line = "{replay} {arrows} {code} {reason} {size}".format(replay=replay, arrows=arrows, code=code, reason=reason, size=size) self.echo(line)
def freeze(self, settings): f = self.parsed.freeze(settings).spec() return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True)))
def __init__(self, data): # type: (bytes) -> Display self.data = data escaped = strutils.bytes_to_escaped_str(data) w = urwid.Text(escaped, wrap="any") super(Display, self).__init__(w)
def spec(self): return strutils.bytes_to_escaped_str(self.val, escape_single_quotes=True)
def freeze(self, settings): g = self.get_generator(settings) return TokValueLiteral(strutils.bytes_to_escaped_str(g[:], escape_single_quotes=True))
def __repr__(self): ret = repr(self.header) if self.payload: ret = ret + "\nPayload:\n" + strutils.bytes_to_escaped_str(self.payload) return ret
def spec(self): inner = strutils.bytes_to_escaped_str(self.val) inner = inner.replace(r"'", r"\x27") return "'" + inner + "'"
def __init__(self, data: bytes): self.data = data escaped = strutils.bytes_to_escaped_str(data) w = urwid.Text(escaped, wrap="any") super().__init__(w)
def __call__(self, data, **metadata): return "Raw", format_text(strutils.bytes_to_escaped_str(data))
def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"foo") == "foo" assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" assert strutils.bytes_to_escaped_str(b"'") == r"'" assert strutils.bytes_to_escaped_str(b'"') == r'"' assert strutils.bytes_to_escaped_str(b"'", escape_single_quotes=True) == r"\'" assert strutils.bytes_to_escaped_str(b'"', escape_single_quotes=True) == r'"' assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") assert strutils.bytes_to_escaped_str(b"\\\\n", True) == "\\ \\ \\ \\ n".replace( " ", "") with tutils.raises(ValueError): strutils.bytes_to_escaped_str(u"such unicode")
def __init__(self, data: bytes): data = strutils.bytes_to_escaped_str(data) w = urwid.Edit(edit_text=data, wrap="any", multiline=True) w = urwid.AttrWrap(w, "editfield") super().__init__(w)
def freeze(self, settings): f = self.parsed.freeze(settings).spec() return self.__class__( TokValueLiteral( strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True)))
def freeze(self, settings): f = self.parsed.freeze(settings).spec() return self.__class__(TokValueLiteral( strutils.bytes_to_escaped_str(f)))
def spec(self): return strutils.bytes_to_escaped_str(self.val)
def __call__(self, data, **metadata): return "Raw", format_text(strutils.bytes_to_escaped_str(data, True))
def __init__(self, data): # type: (bytes) -> Edit data = strutils.bytes_to_escaped_str(data) w = urwid.Edit(edit_text=data, wrap="any", multiline=True) w = urwid.AttrWrap(w, "editfield") super(Edit, self).__init__(w)
def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"foo") == "foo" assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" assert strutils.bytes_to_escaped_str(b"'") == r"'" assert strutils.bytes_to_escaped_str(b'"') == r'"' assert strutils.bytes_to_escaped_str(b"'", escape_single_quotes=True) == r"\'" assert strutils.bytes_to_escaped_str(b'"', escape_single_quotes=True) == r'"' assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") assert strutils.bytes_to_escaped_str(b"\\\\n", True) == "\\ \\ \\ \\ n".replace(" ", "") with tutils.raises(ValueError): strutils.bytes_to_escaped_str(u"such unicode")
def spec(self): inner = strutils.bytes_to_escaped_str(self.val) inner = inner.replace(r"\'", r"\x27") return "'" + inner + "'"
def freeze(self, settings): g = self.get_generator(settings) return TokValueLiteral(strutils.bytes_to_escaped_str(g[:]))
def freeze(self, settings): f = self.parsed.freeze(settings).spec() return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f)))