def _echo_request_line(self, flow):
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
human.format_address(flow.client_conn.address)))
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(GET="green",
DELETE="red").get(method.upper(), "magenta")
method = click.style(strutils.escape_control_characters(method),
fg=method_color,
bold=True)
if ctx.options.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
terminalWidthLimit = max(shutil.get_terminal_size()[0] - 25, 50)
if ctx.options.flow_detail < 1 and len(url) > terminalWidthLimit:
url = url[:terminalWidthLimit] + "…"
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
line = "{client}: {method} {url}{http_version}".format(
client=client, method=method, url=url, http_version=http_version)
self.echo(line)
def _echo_request_line(self, flow):
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
repr(flow.client_conn.address)))
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(GET="green",
DELETE="red").get(method.upper(), "magenta")
method = click.style(strutils.escape_control_characters(method),
fg=method_color,
bold=True)
if self.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
line = "{client}: {method} {url}{http_version}".format(
client=client, method=method, url=url, http_version=http_version)
self.echo(line)
def _echo_request_line(self, flow):
if flow.request.stickycookie:
stickycookie = click.style(
"[stickycookie] ", fg="yellow", bold=True
)
else:
stickycookie = ""
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
repr(flow.client_conn.address)
)
)
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(
GET="green",
DELETE="red"
).get(method.upper(), "magenta")
method = click.style(
strutils.escape_control_characters(method),
fg=method_color,
bold=True
)
if self.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
if len(url) > 200:
url = url[:199] + "…"
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
if self.flow_detail >= 2:
linebreak = "\n "
else:
linebreak = ""
line = "{client}: {linebreak}{stickycookie}{method} {url}{http_version}".format(
client=client,
stickycookie=stickycookie,
linebreak=linebreak,
method=method,
url=url,
http_version=http_version
)
self.echo(line)
def _echo_request_line(self, flow):
if flow.request.stickycookie:
stickycookie = click.style("[stickycookie] ",
fg="yellow",
bold=True)
else:
stickycookie = ""
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
repr(flow.client_conn.address)))
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
method = flow.request.method
method_color = dict(GET="green",
DELETE="red").get(method.upper(), "magenta")
method = click.style(strutils.escape_control_characters(method),
fg=method_color,
bold=True)
if self.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
if len(url) > 200:
url = url[:199] + "…"
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
if self.flow_detail >= 2:
linebreak = "\n "
else:
linebreak = ""
line = "{client}: {linebreak}{stickycookie}{method} {url}{http_version}".format(
client=client,
stickycookie=stickycookie,
linebreak=linebreak,
method=method,
url=url,
http_version=http_version)
self.echo(line)
def _echo_response_line(self, flow: http.HTTPFlow) -> None:
if flow.is_replay == "response":
replay_str = "[replay]"
replay = self.style(replay_str, fg="yellow", bold=True)
else:
replay_str = ""
replay = ""
assert flow.response
code_int = flow.response.status_code
code_color = None
if 200 <= code_int < 300:
code_color = "green"
elif 300 <= code_int < 400:
code_color = "magenta"
elif 400 <= code_int < 600:
code_color = "red"
code = self.style(
str(code_int),
fg=code_color,
bold=True,
blink=(code_int == 418),
)
if not flow.response.is_http2:
reason = flow.response.reason
else:
reason = http.status_codes.RESPONSES.get(flow.response.status_code, "")
reason = self.style(
strutils.escape_control_characters(reason),
fg=code_color,
bold=True
)
if flow.response.raw_content is None:
size = "(content missing)"
else:
size = human.pretty_size(len(flow.response.raw_content))
size = self.style(size, bold=True)
http_version = ""
if (
not (flow.response.is_http10 or flow.response.is_http11)
or flow.request.http_version != flow.response.http_version
):
# Hide version for h1 <-> h1 connections.
http_version = f"{flow.response.http_version} "
arrows = self.style(" <<", bold=True)
if ctx.options.flow_detail == 1:
# This aligns the HTTP response code with the HTTP request method:
# 127.0.0.1:59519: GET http://example.com/
# << 304 Not Modified 0b
pad = max(0,
len(human.format_address(flow.client_conn.peername)) - (2 + len(http_version) + len(replay_str)))
arrows = " " * pad + arrows
self.echo(f"{replay}{arrows} {http_version}{code} {reason} {size}")
def http(self, r):
"""
Performs a single request.
r: A language.http.Request object, or a string representing one
request.
Returns Response if we have a non-ignored response.
May raise a exceptions.NetlibException
"""
logger = log.ConnectionLogger(
self.fp,
self.hexdump,
False,
self.rfile if self.showresp else None,
self.wfile if self.showreq else None,
)
with logger.ctx() as lg:
lg(">> %s" % r)
resp, req = None, None
try:
req = language.serve(r, self.wfile, self.settings)
self.wfile.flush()
# build a dummy request to read the reponse
# ideally this would be returned directly from language.serve
dummy_req = net_http.Request(
first_line_format="relative",
method=req["method"],
scheme=b"http",
host=b"localhost",
port=80,
path=b"/",
http_version=b"HTTP/1.1",
content=b'',
)
resp = self.protocol.read_response(self.rfile, dummy_req)
resp.sslinfo = self.sslinfo
except exceptions.HttpException as v:
lg("Invalid server response: %s" % v)
raise
except exceptions.TcpTimeout:
if self.ignoretimeout:
lg("Timeout (ignored)")
return None
lg("Timeout")
raise
finally:
if resp:
lg("<< %s %s: %s bytes" %
(resp.status_code,
strutils.escape_control_characters(resp.reason)
if resp.reason else "", len(resp.content)))
if resp.status_code in self.ignorecodes:
lg.suppress()
return resp
def _echo_request_line(self, flow: http.HTTPFlow) -> None:
if flow.is_replay == "request":
client = self.style("[replay]", fg="yellow", bold=True)
elif flow.client_conn.peername:
client = self.style(
strutils.escape_control_characters(
human.format_address(flow.client_conn.peername)
)
)
else: # pragma: no cover
# this should not happen, but we're defensive here.
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(
GET="green",
DELETE="red"
).get(method.upper(), "magenta")
method = self.style(
strutils.escape_control_characters(method),
fg=method_color,
bold=True
)
if ctx.options.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
if ctx.options.flow_detail == 1:
# We need to truncate before applying styles, so we just focus on the URL.
terminal_width_limit = max(shutil.get_terminal_size()[0] - 25, 50)
if len(url) > terminal_width_limit:
url = url[:terminal_width_limit] + "…"
url = self.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if (
not (flow.request.is_http10 or flow.request.is_http11)
or flow.request.http_version != getattr(flow.response, "http_version", "HTTP/1.1")
):
# Hide version for h1 <-> h1 connections.
http_version = " " + flow.request.http_version
self.echo(f"{client}: {method} {url}{http_version}")
def http(self, r):
"""
Performs a single request.
r: A language.http.Request object, or a string representing one
request.
Returns Response if we have a non-ignored response.
May raise a exceptions.NetlibException
"""
logger = log.ConnectionLogger(
self.fp,
self.hexdump,
False,
self.rfile if self.showresp else None,
self.wfile if self.showreq else None,
)
with logger.ctx() as lg:
lg(">> %s" % r)
resp, req = None, None
try:
req = language.serve(r, self.wfile, self.settings)
self.wfile.flush()
# build a dummy request to read the reponse
# ideally this would be returned directly from language.serve
dummy_req = net_http.Request(
first_line_format="relative",
method=req["method"],
scheme=b"http",
host=b"localhost",
port=80,
path=b"/",
http_version=b"HTTP/1.1",
content=b'',
)
resp = self.protocol.read_response(self.rfile, dummy_req)
resp.sslinfo = self.sslinfo
except exceptions.HttpException as v:
lg("Invalid server response: %s" % v)
raise
except exceptions.TcpTimeout:
if self.ignoretimeout:
lg("Timeout (ignored)")
return None
lg("Timeout")
raise
finally:
if resp:
lg("<< %s %s: %s bytes" % (
resp.status_code, strutils.escape_control_characters(resp.reason) if resp.reason else "", len(resp.content)
))
if resp.status_code in self.ignorecodes:
lg.suppress()
return resp
def _echo_request_line(self, flow):
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
human.format_address(flow.client_conn.address)
)
)
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(
GET="green",
DELETE="red"
).get(method.upper(), "magenta")
method = click.style(
strutils.escape_control_characters(method),
fg=method_color,
bold=True
)
if ctx.options.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
terminalWidthLimit = max(shutil.get_terminal_size()[0] - 25, 50)
if ctx.options.flow_detail < 1 and len(url) > terminalWidthLimit:
url = url[:terminalWidthLimit] + "…"
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
line = "{client}: {method} {url}{http_version}".format(
client=client,
method=method,
url=url,
http_version=http_version
)
self.echo(line)
def dump(self, data, hexdump):
if hexdump:
for line in strutils.hexdump(data):
self("\t%s %s %s" % line)
else:
data = strutils.native(
strutils.escape_control_characters(
data.decode("ascii", "replace").replace(u"\ufffd", u".")))
for i in data.split("\n"):
self("\t%s" % i)
def safe_to_print(lines, encoding="utf8"):
"""
Wraps a content generator so that each text portion is a *safe to print* unicode string.
"""
for line in lines:
clean_line = []
for (style, text) in line:
if isinstance(text, bytes):
text = text.decode(encoding, "replace")
text = strutils.escape_control_characters(text)
clean_line.append((style, text))
yield clean_line
def echo_flow(self, f):
if f.request:
self._echo_request_line(f)
self._echo_message(f.request)
if f.response:
self._echo_response_line(f)
self._echo_message(f.response)
if f.error:
msg = strutils.escape_control_characters(f.error.msg)
self.echo(" << {}".format(msg), bold=True, fg="red")
def echo_flow(self, f):
if f.request:
self._echo_request_line(f)
self._echo_message(f.request)
if f.response:
self._echo_response_line(f)
self._echo_message(f.response)
if f.error:
msg = strutils.escape_control_characters(f.error.msg)
self.echo(" << {}".format(msg), bold=True, fg="red")
def test_escape_control_characters():
assert strutils.escape_control_characters(u"one") == u"one"
assert strutils.escape_control_characters(u"\00ne") == u".ne"
assert strutils.escape_control_characters(u"\nne") == u"\nne"
assert strutils.escape_control_characters(u"\nne", False) == u".ne"
assert strutils.escape_control_characters(u"\u2605") == u"\u2605"
assert (
strutils.escape_control_characters(bytes(bytearray(range(128))).decode()) ==
u'.........\t\n..\r.................. !"#$%&\'()*+,-./0123456789:;<'
u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.'
)
assert (
strutils.escape_control_characters(bytes(bytearray(range(128))).decode(), False) ==
u'................................ !"#$%&\'()*+,-./0123456789:;<'
u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.'
)
with tutils.raises(ValueError):
strutils.escape_control_characters(b"foo")
def test_escape_control_characters():
assert strutils.escape_control_characters("one") == "one"
assert strutils.escape_control_characters("\00ne") == ".ne"
assert strutils.escape_control_characters("\nne") == "\nne"
assert strutils.escape_control_characters("\nne", False) == ".ne"
assert strutils.escape_control_characters("\u2605") == "\u2605"
assert (
strutils.escape_control_characters(
bytes(bytearray(range(128))).decode()) ==
'.........\t\n..\r.................. !"#$%&\'()*+,-./0123456789:;<'
'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.')
assert (
strutils.escape_control_characters(
bytes(bytearray(range(128))).decode(), False) ==
'................................ !"#$%&\'()*+,-./0123456789:;<'
'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.')
with pytest.raises(ValueError):
strutils.escape_control_characters(b"foo")
def dump(self, data, hexdump):
if hexdump:
for line in strutils.hexdump(data):
self("\t%s %s %s" % line)
else:
data = strutils.native(
strutils.escape_control_characters(
data
.decode("ascii", "replace")
.replace(u"\ufffd", u".")
)
)
for i in data.split("\n"):
self("\t%s" % i)
def _echo_response_line(self, flow):
if flow.response.is_replay:
replay = click.style("[replay] ", fg="yellow", bold=True)
else:
replay = ""
code = flow.response.status_code
code_color = None
if 200 <= code < 300:
code_color = "green"
elif 300 <= code < 400:
code_color = "magenta"
elif 400 <= code < 600:
code_color = "red"
code = click.style(
str(code),
fg=code_color,
bold=True,
blink=(code == 418)
)
reason = click.style(
strutils.escape_control_characters(flow.response.reason),
fg=code_color,
bold=True
)
if flow.response.raw_content is None:
size = "(content missing)"
else:
size = human.pretty_size(len(flow.response.raw_content))
size = click.style(size, bold=True)
arrows = click.style(" <<", bold=True)
if self.flow_detail == 1:
# This aligns the HTTP response code with the HTTP request method:
# 127.0.0.1:59519: GET http://example.com/
# << 304 Not Modified 0b
arrows = " " * (len(repr(flow.client_conn.address)) - 2) + arrows
line = "{replay}{arrows} {code} {reason} {size}".format(
replay=replay,
arrows=arrows,
code=code,
reason=reason,
size=size
)
self.echo(line)
def http(self, r):
"""
Performs a single request.
r: A language.http.Request object, or a string representing one
request.
Returns Response if we have a non-ignored response.
May raise a exceptions.NetlibException
"""
logger = log.ConnectionLogger(
self.fp,
self.hexdump,
False,
self.rfile if self.showresp else None,
self.wfile if self.showreq else None,
)
with logger.ctx() as lg:
lg(">> %s" % r)
resp, req = None, None
try:
req = language.serve(r, self.wfile, self.settings)
self.wfile.flush()
resp = self.protocol.read_response(
self.rfile, treq(method=req["method"].encode()))
resp.sslinfo = self.sslinfo
except exceptions.HttpException as v:
lg("Invalid server response: %s" % v)
raise
except exceptions.TcpTimeout:
if self.ignoretimeout:
lg("Timeout (ignored)")
return None
lg("Timeout")
raise
finally:
if resp:
lg("<< %s %s: %s bytes" %
(resp.status_code,
strutils.escape_control_characters(resp.reason)
if resp.reason else "", len(resp.content)))
if resp.status_code in self.ignorecodes:
lg.suppress()
return resp
def echo_flow(self, f):
if f.request:
self._echo_request_line(f)
if ctx.options.flow_detail >= 2:
self._echo_headers(f.request.headers)
if ctx.options.flow_detail >= 3:
self._echo_message(f.request)
if f.response:
self._echo_response_line(f)
if ctx.options.flow_detail >= 2:
self._echo_headers(f.response.headers)
if ctx.options.flow_detail >= 3:
self._echo_message(f.response)
if f.error:
msg = strutils.escape_control_characters(f.error.msg)
self.echo(" << {}".format(msg), bold=True, fg="red")
def echo_flow(self, f):
if f.request:
self._echo_request_line(f)
if ctx.options.flow_detail >= 2:
self._echo_headers(f.request.headers)
if ctx.options.flow_detail >= 3:
self._echo_message(f.request, f)
if f.response:
self._echo_response_line(f)
if ctx.options.flow_detail >= 2:
self._echo_headers(f.response.headers)
if ctx.options.flow_detail >= 3:
self._echo_message(f.response, f)
if f.error:
msg = strutils.escape_control_characters(f.error.msg)
self.echo(" << {}".format(msg), bold=True, fg="red")
def http(self, r):
"""
Performs a single request.
r: A language.http.Request object, or a string representing one
request.
Returns Response if we have a non-ignored response.
May raise a exceptions.NetlibException
"""
logger = log.ConnectionLogger(
self.fp,
self.hexdump,
False,
self.rfile if self.showresp else None,
self.wfile if self.showreq else None,
)
with logger.ctx() as lg:
lg(">> %s" % r)
resp, req = None, None
try:
req = language.serve(r, self.wfile, self.settings)
self.wfile.flush()
resp = self.protocol.read_response(self.rfile, treq(method=req["method"].encode()))
resp.sslinfo = self.sslinfo
except exceptions.HttpException as v:
lg("Invalid server response: %s" % v)
raise
except exceptions.TcpTimeout:
if self.ignoretimeout:
lg("Timeout (ignored)")
return None
lg("Timeout")
raise
finally:
if resp:
lg("<< %s %s: %s bytes" % (
resp.status_code, strutils.escape_control_characters(resp.reason) if resp.reason else "", len(resp.content)
))
if resp.status_code in self.ignorecodes:
lg.suppress()
return resp
def echo_flow(self, f: http.HTTPFlow) -> None:
if f.request:
self._echo_request_line(f)
if ctx.options.flow_detail >= 2:
self._echo_headers(f.request.headers)
if ctx.options.flow_detail >= 3:
self._echo_message(f.request, f)
if ctx.options.flow_detail >= 2:
self._echo_trailers(f.request.trailers)
if f.response:
self._echo_response_line(f)
if ctx.options.flow_detail >= 2:
self._echo_headers(f.response.headers)
if ctx.options.flow_detail >= 3:
self._echo_message(f.response, f)
if ctx.options.flow_detail >= 2:
self._echo_trailers(f.response.trailers)
if f.error:
msg = strutils.escape_control_characters(f.error.msg)
self.echo(f" << {msg}", bold=True, fg="red")
