def websocket_start(self, f):
if ctx.options.stream_websockets:
f.stream = True
ctx.log.info(
"Streaming WebSocket messages between {client} and {server}".
format(client=human.format_address(f.client_conn.address),
server=human.format_address(f.server_conn.address)))
def receive_handshake_data(
self,
data: bytes) -> layer.CommandGenerator[Tuple[bool, Optional[str]]]:
if not self.send_connect:
return (yield from super().receive_handshake_data(data))
self.buf += data
response_head = self.buf.maybe_extract_lines()
if response_head:
response_head = [
bytes(x) for x in response_head
] # TODO: Make url.parse compatible with bytearrays
try:
response = http1.read_response_head(response_head)
except ValueError as e:
proxyaddr = human.format_address(
self.tunnel_connection.address)
yield commands.Log(f"{proxyaddr}: {e}")
return False, f"Error connecting to {proxyaddr}: {e}"
if 200 <= response.status_code < 300:
if self.buf:
yield from self.receive_data(bytes(self.buf))
del self.buf
return True, None
else:
proxyaddr = human.format_address(
self.tunnel_connection.address)
raw_resp = b"\n".join(response_head)
yield commands.Log(f"{proxyaddr}: {raw_resp!r}", level="debug")
return False, f"Upstream proxy {proxyaddr} refused HTTP CONNECT request: {response.status_code} {response.reason}"
else:
return False, None
def test_format_address():
assert human.format_address(("::1", "54010", "0", "0")) == "[::1]:54010"
assert human.format_address(
("::ffff:127.0.0.1", "54010", "0", "0")) == "127.0.0.1:54010"
assert human.format_address(("127.0.0.1", "54010")) == "127.0.0.1:54010"
assert human.format_address(
("example.com", "54010")) == "example.com:54010"
def websocket_start(self, f):
if ctx.options.stream_websockets:
f.stream = True
ctx.log.info("Streaming WebSocket messages between {client} and {server}".format(
client=human.format_address(f.client_conn.address),
server=human.format_address(f.server_conn.address))
)
def message_info(self, message: WebSocketMessage) -> str:
return "{client} {direction} WebSocket {type} message {direction} {server}{endpoint}".format(
type=message.type,
client=human.format_address(self.client_conn.address),
server=human.format_address(self.server_conn.address),
direction="->" if message.from_client else "<-",
endpoint=self.handshake_flow.request.path,
)
def message_info(self, message: WebSocketMessage) -> str:
return "{client} {direction} WebSocket {type} message {direction} {server}{endpoint}".format(
type=message.type,
client=human.format_address(self.client_conn.address),
server=human.format_address(self.server_conn.address),
direction="->" if message.from_client else "<-",
endpoint=self.handshake_flow.request.path,
)
def tcp_message(self, f):
if self.match(f):
message = f.messages[-1]
direction = "->" if message.from_client else "<-"
self.echo("{client} {direction} tcp {direction} {server}".format(
client=human.format_address(f.client_conn.peername),
server=human.format_address(f.server_conn.address),
direction=direction,
))
if ctx.options.flow_detail >= 3:
self._echo_message(message, f)
def tcp_message(self, f):
if self.match(f):
message = f.messages[-1]
direction = "->" if message.from_client else "<-"
self.echo("{client} {direction} tcp {direction} {server}".format(
client=human.format_address(f.client_conn.address),
server=human.format_address(f.server_conn.address),
direction=direction,
))
if ctx.options.flow_detail >= 3:
self._echo_message(message)
def tcp_error(self, f):
self.echo(
"Error in TCP connection to {}: {}".format(
human.format_address(f.server_conn.address), f.error
),
fg="red"
)
def websocket_error(self, f):
self.echo(
"Error in WebSocket connection to {}: {}".format(
human.format_address(f.server_conn.address), f.error
),
fg="red"
)
def running(self):
if ctx.options.server:
ctx.log.info(
"Proxy server listening at http://{}".format(
human.format_address(ctx.master.server.address)
)
)
def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:
if self.conn.sni:
dest = self.conn.sni
else:
dest = human.format_address(self.context.server.address)
level: Literal["warn", "info"] = "warn"
if err.startswith("Cannot parse ClientHello"):
pass
elif "unknown ca" in err or "bad certificate" in err:
err = f"The client does not trust the proxy's certificate for {dest} ({err})"
elif err == "connection closed":
err = (
f"The client disconnected during the handshake. If this happens consistently for {dest}, "
f"this may indicate that the client does not trust the proxy's certificate."
)
level = "info"
elif err == "connection closed early":
pass
else:
err = f"The client may not trust the proxy's certificate for {dest} ({err})"
if err != "connection closed early":
yield commands.Log(f"Client TLS handshake failed. {err}",
level=level)
yield from super().on_handshake_error(err)
self.event_to_child = self.errored # type: ignore
def run(self):
self.event_loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.event_loop)
self.tmaster = self.masterclass(self.options)
self.tmaster.addons.add(core.Core())
self.name = "ProxyThread (%s)" % human.format_address(self.tmaster.server.address)
self.tmaster.run()
def websocket_error(self, f):
self.echo(
"Error in WebSocket connection to {}: {}".format(
human.format_address(f.server_conn.address), f.error
),
fg="red"
)
def tcp_error(self, f):
self.echo(
"Error in TCP connection to {}: {}".format(
human.format_address(f.server_conn.address), f.error
),
fg="red"
)
def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:
if self.conn.sni:
dest = self.conn.sni
else:
dest = human.format_address(self.context.server.address)
level: Literal["warn", "info"] = "warn"
if err.startswith("Cannot parse ClientHello"):
pass
elif "('SSL routines', 'tls_early_post_process_client_hello', 'unsupported protocol')" in err:
err = (
f"Client and mitmproxy cannot agree on a TLS version to use. "
f"You may need to adjust mitmproxy's tls_version_client_min option."
)
elif "unknown ca" in err or "bad certificate" in err or "certificate unknown" in err:
err = f"The client does not trust the proxy's certificate for {dest} ({err})"
elif err == "connection closed":
err = (
f"The client disconnected during the handshake. If this happens consistently for {dest}, "
f"this may indicate that the client does not trust the proxy's certificate."
)
level = "info"
elif err == "connection closed early":
pass
else:
err = f"The client may not trust the proxy's certificate for {dest} ({err})"
if err != "connection closed early":
yield commands.Log(f"Client TLS handshake failed. {err}",
level=level)
yield from super().on_handshake_error(err)
self.event_to_child = self.errored # type: ignore
def generate(self, f: mitmproxy.flow.Flow) -> str:
if isinstance(f, http.HTTPFlow):
return f.request.url
elif isinstance(f, tcp.TCPFlow):
return human.format_address(f.server_conn.address)
else:
raise NotImplementedError()
def _echo_request_line(self, flow):
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
human.format_address(flow.client_conn.address)))
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(GET="green",
DELETE="red").get(method.upper(), "magenta")
method = click.style(strutils.escape_control_characters(method),
fg=method_color,
bold=True)
if ctx.options.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
terminalWidthLimit = max(shutil.get_terminal_size()[0] - 25, 50)
if ctx.options.flow_detail < 1 and len(url) > terminalWidthLimit:
url = url[:terminalWidthLimit] + "…"
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
line = "{client}: {method} {url}{http_version}".format(
client=client, method=method, url=url, http_version=http_version)
self.echo(line)
def _echo_response_line(self, flow: http.HTTPFlow) -> None:
if flow.is_replay == "response":
replay_str = "[replay]"
replay = self.style(replay_str, fg="yellow", bold=True)
else:
replay_str = ""
replay = ""
assert flow.response
code_int = flow.response.status_code
code_color = None
if 200 <= code_int < 300:
code_color = "green"
elif 300 <= code_int < 400:
code_color = "magenta"
elif 400 <= code_int < 600:
code_color = "red"
code = self.style(
str(code_int),
fg=code_color,
bold=True,
blink=(code_int == 418),
)
if not flow.response.is_http2:
reason = flow.response.reason
else:
reason = http.status_codes.RESPONSES.get(flow.response.status_code, "")
reason = self.style(
strutils.escape_control_characters(reason),
fg=code_color,
bold=True
)
if flow.response.raw_content is None:
size = "(content missing)"
else:
size = human.pretty_size(len(flow.response.raw_content))
size = self.style(size, bold=True)
http_version = ""
if (
not (flow.response.is_http10 or flow.response.is_http11)
or flow.request.http_version != flow.response.http_version
):
# Hide version for h1 <-> h1 connections.
http_version = f"{flow.response.http_version} "
arrows = self.style(" <<", bold=True)
if ctx.options.flow_detail == 1:
# This aligns the HTTP response code with the HTTP request method:
# 127.0.0.1:59519: GET http://example.com/
# << 304 Not Modified 0b
pad = max(0,
len(human.format_address(flow.client_conn.peername)) - (2 + len(http_version) + len(replay_str)))
arrows = " " * pad + arrows
self.echo(f"{replay}{arrows} {http_version}{code} {reason} {size}")
def test_format_address():
assert human.format_address(("::1", "54010", "0", "0")) == "[::1]:54010"
assert human.format_address(("::ffff:127.0.0.1", "54010", "0", "0")) == "127.0.0.1:54010"
assert human.format_address(("127.0.0.1", "54010")) == "127.0.0.1:54010"
assert human.format_address(("example.com", "54010")) == "example.com:54010"
assert human.format_address(("::", "8080")) == "*:8080"
assert human.format_address(("0.0.0.0", "8080")) == "*:8080"
assert human.format_address(None) == "<no address>"
def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:
if self.conn.sni:
dest = self.conn.sni
else:
dest = human.format_address(self.context.server.address)
if err.startswith("Cannot parse ClientHello"):
pass
elif "unknown ca" in err or "bad certificate" in err:
err = f"The client does not trust the proxy's certificate for {dest} ({err})"
else:
err = f"The client may not trust the proxy's certificate for {dest} ({err})"
yield commands.Log(f"Client TLS handshake failed. {err}", level="warn")
yield from super().on_handshake_error(err)
def _echo_response_line(self, flow):
if flow.response.is_replay:
replay = click.style("[replay] ", fg="yellow", bold=True)
else:
replay = ""
code = flow.response.status_code
code_color = None
if 200 <= code < 300:
code_color = "green"
elif 300 <= code < 400:
code_color = "magenta"
elif 400 <= code < 600:
code_color = "red"
code = click.style(
str(code),
fg=code_color,
bold=True,
blink=(code == 418)
)
reason = click.style(
strutils.escape_control_characters(flow.response.reason),
fg=code_color,
bold=True
)
if flow.response.raw_content is None:
size = "(content missing)"
else:
size = human.pretty_size(len(flow.response.raw_content))
size = click.style(size, bold=True)
arrows = click.style(" <<", bold=True)
if ctx.options.flow_detail == 1:
# This aligns the HTTP response code with the HTTP request method:
# 127.0.0.1:59519: GET http://example.com/
# << 304 Not Modified 0b
arrows = " " * (len(human.format_address(flow.client_conn.address)) - 2) + arrows
line = "{replay}{arrows} {code} {reason} {size}".format(
replay=replay,
arrows=arrows,
code=code,
reason=reason,
size=size
)
self.echo(line)
def _echo_response_line(self, flow):
if flow.response.is_replay:
replay = click.style("[replay] ", fg="yellow", bold=True)
else:
replay = ""
code = flow.response.status_code
code_color = None
if 200 <= code < 300:
code_color = "green"
elif 300 <= code < 400:
code_color = "magenta"
elif 400 <= code < 600:
code_color = "red"
code = click.style(
str(code),
fg=code_color,
bold=True,
blink=(code == 418)
)
reason = click.style(
strutils.escape_control_characters(flow.response.reason),
fg=code_color,
bold=True
)
if flow.response.raw_content is None:
size = "(content missing)"
else:
size = human.pretty_size(len(flow.response.raw_content))
size = click.style(size, bold=True)
arrows = click.style(" <<", bold=True)
if ctx.options.flow_detail == 1:
# This aligns the HTTP response code with the HTTP request method:
# 127.0.0.1:59519: GET http://example.com/
# << 304 Not Modified 0b
arrows = " " * (len(human.format_address(flow.client_conn.address)) - 2) + arrows
line = "{replay}{arrows} {code} {reason} {size}".format(
replay=replay,
arrows=arrows,
code=code,
reason=reason,
size=size
)
self.echo(line)
def __repr__(self):
if self.tls_established:
tls = f"[{self.tls_version}] "
else:
tls = ""
if self.alpn_proto_negotiated:
alpn = "[ALPN: {}] ".format(
strutils.bytes_to_escaped_str(self.alpn_proto_negotiated))
else:
alpn = ""
return "<ClientConnection: {tls}{alpn}{address}>".format(
tls=tls,
alpn=alpn,
address=human.format_address(self.address),
)
def __repr__(self):
if self.tls_established and self.sni:
tls = "[{}: {}] ".format(self.tls_version or "TLS", self.sni)
elif self.tls_established:
tls = "[{}] ".format(self.tls_version or "TLS")
else:
tls = ""
if self.alpn_proto_negotiated:
alpn = "[ALPN: {}] ".format(
strutils.bytes_to_escaped_str(self.alpn_proto_negotiated))
else:
alpn = ""
return "<ServerConnection: {tls}{alpn}{address}>".format(
tls=tls,
alpn=alpn,
address=human.format_address(self.address),
)
def _echo_request_line(self, flow: http.HTTPFlow) -> None:
if flow.is_replay == "request":
client = self.style("[replay]", fg="yellow", bold=True)
elif flow.client_conn.peername:
client = self.style(
strutils.escape_control_characters(
human.format_address(flow.client_conn.peername)
)
)
else: # pragma: no cover
# this should not happen, but we're defensive here.
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(
GET="green",
DELETE="red"
).get(method.upper(), "magenta")
method = self.style(
strutils.escape_control_characters(method),
fg=method_color,
bold=True
)
if ctx.options.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
if ctx.options.flow_detail == 1:
# We need to truncate before applying styles, so we just focus on the URL.
terminal_width_limit = max(shutil.get_terminal_size()[0] - 25, 50)
if len(url) > terminal_width_limit:
url = url[:terminal_width_limit] + "…"
url = self.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if (
not (flow.request.is_http10 or flow.request.is_http11)
or flow.request.http_version != getattr(flow.response, "http_version", "HTTP/1.1")
):
# Hide version for h1 <-> h1 connections.
http_version = " " + flow.request.http_version
self.echo(f"{client}: {method} {url}{http_version}")
def __repr__(self):
if self.tls_established:
tls = "[{}] ".format(self.tls_version)
else:
tls = ""
if self.alpn_proto_negotiated:
alpn = "[ALPN: {}] ".format(
strutils.bytes_to_escaped_str(self.alpn_proto_negotiated)
)
else:
alpn = ""
return "<ClientConnection: {tls}{alpn}{address}>".format(
tls=tls,
alpn=alpn,
address=human.format_address(self.address),
)
def _echo_request_line(self, flow):
if flow.client_conn:
client = click.style(
strutils.escape_control_characters(
human.format_address(flow.client_conn.address)
)
)
elif flow.request.is_replay:
client = click.style("[replay]", fg="yellow", bold=True)
else:
client = ""
pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else ''
method = flow.request.method + pushed
method_color = dict(
GET="green",
DELETE="red"
).get(method.upper(), "magenta")
method = click.style(
strutils.escape_control_characters(method),
fg=method_color,
bold=True
)
if ctx.options.showhost:
url = flow.request.pretty_url
else:
url = flow.request.url
terminalWidthLimit = max(shutil.get_terminal_size()[0] - 25, 50)
if ctx.options.flow_detail < 1 and len(url) > terminalWidthLimit:
url = url[:terminalWidthLimit] + "…"
url = click.style(strutils.escape_control_characters(url), bold=True)
http_version = ""
if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
# We hide "normal" HTTP 1.
http_version = " " + flow.request.http_version
line = "{client}: {method} {url}{http_version}".format(
client=client,
method=method,
url=url,
http_version=http_version
)
self.echo(line)
def running():
logger.info('FlowInterceptor is running, proxy server listening at http://{}'.format(
human.format_address(ctx.master.server.address)
))
async def open_connection(self, command: commands.OpenConnection) -> None:
if not command.connection.address:
self.log(f"Cannot open connection, no hostname given.")
self.server_event(events.OpenConnectionCompleted(command, f"Cannot open connection, no hostname given."))
return
hook_data = server_hooks.ServerConnectionHookData(
client=self.client,
server=command.connection
)
await self.handle_hook(server_hooks.ServerConnectHook(hook_data))
if command.connection.error:
self.log(f"server connection to {human.format_address(command.connection.address)} killed before connect.")
self.server_event(events.OpenConnectionCompleted(command, "Connection killed."))
return
async with self.max_conns[command.connection.address]:
try:
command.connection.timestamp_start = time.time()
reader, writer = await asyncio.open_connection(*command.connection.address)
except (IOError, asyncio.CancelledError) as e:
err = str(e)
if not err: # str(CancelledError()) returns empty string.
err = "connection cancelled"
self.log(f"error establishing server connection: {err}")
command.connection.error = err
self.server_event(events.OpenConnectionCompleted(command, err))
if isinstance(e, asyncio.CancelledError):
# From https://docs.python.org/3/library/asyncio-exceptions.html#asyncio.CancelledError:
# > In almost all situations the exception must be re-raised.
# It is not really defined what almost means here, but we play safe.
raise
else:
command.connection.timestamp_tcp_setup = time.time()
command.connection.state = ConnectionState.OPEN
command.connection.peername = writer.get_extra_info('peername')
command.connection.sockname = writer.get_extra_info('sockname')
self.transports[command.connection].reader = reader
self.transports[command.connection].writer = writer
assert command.connection.peername
if command.connection.address[0] != command.connection.peername[0]:
addr = f"{command.connection.address[0]} ({human.format_address(command.connection.peername)})"
else:
addr = human.format_address(command.connection.address)
self.log(f"server connect {addr}")
connected_hook = asyncio_utils.create_task(
self.handle_hook(server_hooks.ServerConnectedHook(hook_data)),
name=f"handle_hook(server_connected) {addr}",
client=self.client.peername,
)
if not connected_hook:
return # this should not be needed, see asyncio_utils.create_task
self.server_event(events.OpenConnectionCompleted(command, None))
# during connection opening, this function is the designated handler that can be cancelled.
# once we have a connection, we do want the teardown here to happen in any case, so we
# reassign the handler to .handle_connection and then clean up here once that is done.
new_handler = asyncio_utils.create_task(
self.handle_connection(command.connection),
name=f"server connection handler for {addr}",
client=self.client.peername,
)
if not new_handler:
return # this should not be needed, see asyncio_utils.create_task
self.transports[command.connection].handler = new_handler
await asyncio.wait([new_handler])
self.log(f"server disconnect {addr}")
command.connection.timestamp_end = time.time()
await connected_hook # wait here for this so that closed always comes after connected.
await self.handle_hook(server_hooks.ServerDisconnectedHook(hook_data))
def log(self, msg, level):
msg = "{}: {}".format(human.format_address(self.client_conn.address), msg)
self.channel.tell("log", log.LogEntry(msg, level))
def running(self):
if ctx.master.server.bound:
ctx.log.info("代理服务器监听端口在： http://{}".format(
human.format_address(ctx.master.server.address)))
ctx.log.info("--------------------")
def flowdetails(state, flow: mitmproxy.flow.Flow):
text = []
sc = flow.server_conn
cc = flow.client_conn
req: typing.Optional[http.Request]
resp: typing.Optional[http.Response]
if isinstance(flow, http.HTTPFlow):
req = flow.request
resp = flow.response
else:
req = None
resp = None
metadata = flow.metadata
if metadata is not None and len(metadata) > 0:
parts = [(str(k), repr(v)) for k, v in metadata.items()]
text.append(urwid.Text([("head", "Metadata:")]))
text.extend(common.format_keyvals(parts, indent=4))
if sc is not None and sc.peername:
text.append(urwid.Text([("head", "Server Connection:")]))
parts = [
("Address", human.format_address(sc.address)),
]
if sc.peername:
parts.append(("Resolved Address", human.format_address(sc.peername)))
if resp:
parts.append(("HTTP Version", resp.http_version))
if sc.alpn:
parts.append(("ALPN", strutils.bytes_to_escaped_str(sc.alpn)))
text.extend(
common.format_keyvals(parts, indent=4)
)
if sc.certificate_list:
c = sc.certificate_list[0]
text.append(urwid.Text([("head", "Server Certificate:")]))
parts = [
("Type", "%s, %s bits" % c.keyinfo),
("SHA256 digest", c.fingerprint().hex()),
("Valid to", str(c.notafter)),
("Valid from", str(c.notbefore)),
("Serial", str(c.serial)),
("Subject", urwid.Pile(common.format_keyvals(c.subject, key_format="highlight"))),
("Issuer", urwid.Pile(common.format_keyvals(c.issuer, key_format="highlight")))
]
if c.altnames:
parts.append(("Alt names", ", ".join(c.altnames)))
text.extend(
common.format_keyvals(parts, indent=4)
)
if cc is not None:
text.append(urwid.Text([("head", "Client Connection:")]))
parts = [
("Address", human.format_address(cc.peername)),
]
if req:
parts.append(("HTTP Version", req.http_version))
if cc.tls_version:
parts.append(("TLS Version", cc.tls_version))
if cc.sni:
parts.append(("Server Name Indication", cc.sni))
if cc.cipher:
parts.append(("Cipher Name", cc.cipher))
if cc.alpn:
parts.append(("ALPN", strutils.bytes_to_escaped_str(cc.alpn)))
text.extend(
common.format_keyvals(parts, indent=4)
)
parts = []
if cc is not None and cc.timestamp_start:
parts.append(
(
"Client conn. established",
maybe_timestamp(cc, "timestamp_start")
)
)
if cc.tls_established:
parts.append(
(
"Client conn. TLS handshake",
maybe_timestamp(cc, "timestamp_tls_setup")
)
)
parts.append(
(
"Client conn. closed",
maybe_timestamp(cc, "timestamp_end")
)
)
if sc is not None and sc.timestamp_start:
parts.append(
(
"Server conn. initiated",
maybe_timestamp(sc, "timestamp_start")
)
)
parts.append(
(
"Server conn. TCP handshake",
maybe_timestamp(sc, "timestamp_tcp_setup")
)
)
if sc.tls_established:
parts.append(
(
"Server conn. TLS handshake",
maybe_timestamp(sc, "timestamp_tls_setup")
)
)
parts.append(
(
"Server conn. closed",
maybe_timestamp(sc, "timestamp_end")
)
)
if req is not None and req.timestamp_start:
parts.append(
(
"First request byte",
maybe_timestamp(req, "timestamp_start")
)
)
parts.append(
(
"Request complete",
maybe_timestamp(req, "timestamp_end")
)
)
if resp is not None and resp.timestamp_start:
parts.append(
(
"First response byte",
maybe_timestamp(resp, "timestamp_start")
)
)
parts.append(
(
"Response complete",
maybe_timestamp(resp, "timestamp_end")
)
)
if parts:
# sort operations by timestamp
parts = sorted(parts, key=lambda p: p[1])
text.append(urwid.Text([("head", "Timing:")]))
text.extend(common.format_keyvals(parts, indent=4))
return searchable.Searchable(text)
def running(self):
if self.server:
ctx.log.info("Proxy server listening at http://{}".format(
human.format_address(ctx.master.server.address)))
def flowdetails(state, flow: http.HTTPFlow):
text = []
sc = flow.server_conn
cc = flow.client_conn
req = flow.request
resp = flow.response
metadata = flow.metadata
if metadata is not None and len(metadata) > 0:
parts = [[str(k), repr(v)] for k, v in metadata.items()]
text.append(urwid.Text([("head", "Metadata:")]))
text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
if sc is not None and sc.ip_address:
text.append(urwid.Text([("head", "Server Connection:")]))
parts = [
["Address", human.format_address(sc.address)],
]
if sc.ip_address:
parts.append(["Resolved Address", human.format_address(sc.ip_address)])
if resp:
parts.append(["HTTP Version", resp.http_version])
if sc.alpn_proto_negotiated:
parts.append(["ALPN", sc.alpn_proto_negotiated])
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4)
)
c = sc.cert
if c:
text.append(urwid.Text([("head", "Server Certificate:")]))
parts = [
["Type", "%s, %s bits" % c.keyinfo],
["SHA1 digest", c.digest("sha1")],
["Valid to", str(c.notafter)],
["Valid from", str(c.notbefore)],
["Serial", str(c.serial)],
[
"Subject",
urwid.BoxAdapter(
urwid.ListBox(
common.format_keyvals(
c.subject,
key="highlight",
val="text"
)
),
len(c.subject)
)
],
[
"Issuer",
urwid.BoxAdapter(
urwid.ListBox(
common.format_keyvals(
c.issuer, key="highlight", val="text"
)
),
len(c.issuer)
)
]
]
if c.altnames:
parts.append(
[
"Alt names",
", ".join(strutils.bytes_to_escaped_str(x) for x in c.altnames)
]
)
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4)
)
if cc is not None:
text.append(urwid.Text([("head", "Client Connection:")]))
parts = [
["Address", "{}:{}".format(cc.address[0], cc.address[1])],
]
if req:
parts.append(["HTTP Version", req.http_version])
if cc.tls_version:
parts.append(["TLS Version", cc.tls_version])
if cc.sni:
parts.append(["Server Name Indication", cc.sni])
if cc.cipher_name:
parts.append(["Cipher Name", cc.cipher_name])
if cc.alpn_proto_negotiated:
parts.append(["ALPN", cc.alpn_proto_negotiated])
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4)
)
parts = []
if cc is not None and cc.timestamp_start:
parts.append(
[
"Client conn. established",
maybe_timestamp(cc, "timestamp_start")
]
)
if cc.ssl_established:
parts.append(
[
"Client conn. TLS handshake",
maybe_timestamp(cc, "timestamp_ssl_setup")
]
)
if sc is not None and sc.timestamp_start:
parts.append(
[
"Server conn. initiated",
maybe_timestamp(sc, "timestamp_start")
]
)
parts.append(
[
"Server conn. TCP handshake",
maybe_timestamp(sc, "timestamp_tcp_setup")
]
)
if sc.ssl_established:
parts.append(
[
"Server conn. TLS handshake",
maybe_timestamp(sc, "timestamp_ssl_setup")
]
)
if req is not None and req.timestamp_start:
parts.append(
[
"First request byte",
maybe_timestamp(req, "timestamp_start")
]
)
parts.append(
[
"Request complete",
maybe_timestamp(req, "timestamp_end")
]
)
if resp is not None and resp.timestamp_start:
parts.append(
[
"First response byte",
maybe_timestamp(resp, "timestamp_start")
]
)
parts.append(
[
"Response complete",
maybe_timestamp(resp, "timestamp_end")
]
)
if parts:
# sort operations by timestamp
parts = sorted(parts, key=lambda p: p[1])
text.append(urwid.Text([("head", "Timing:")]))
text.extend(common.format_keyvals(parts, key="key", val="text", indent=4))
return searchable.Searchable(text)
class ConnectionHandler(metaclass=abc.ABCMeta):
transports: typing.MutableMapping[Connection, ConnectionIO]
timeout_watchdog: TimeoutWatchdog
client: Client
max_conns: typing.DefaultDict[Address, asyncio.Semaphore]
layer: layer.Layer
def __init__(self, context: Context) -> None:
self.client = context.client
self.transports = {}
self.max_conns = collections.defaultdict(lambda: asyncio.Semaphore(5))
# Ask for the first layer right away.
# In a reverse proxy scenario, this is necessary as we would otherwise hang
# on protocols that start with a server greeting.
self.layer = layer.NextLayer(context, ask_on_start=True)
self.timeout_watchdog = TimeoutWatchdog(self.on_timeout)
async def handle_client(self) -> None:
watch = asyncio_utils.create_task(
self.timeout_watchdog.watch(),
name="timeout watchdog",
client=self.client.peername,
)
if not watch:
return # this should not be needed, see asyncio_utils.create_task
self.log("client connect")
await self.handle_hook(server_hooks.ClientConnectedHook(self.client))
if self.client.error:
self.log("client kill connection")
writer = self.transports.pop(self.client).writer
assert writer
writer.close()
else:
handler = asyncio_utils.create_task(
self.handle_connection(self.client),
name=f"client connection handler",
client=self.client.peername,
)
if not handler:
return # this should not be needed, see asyncio_utils.create_task
self.transports[self.client].handler = handler
self.server_event(events.Start())
await asyncio.wait([handler])
watch.cancel()
self.log("client disconnect")
self.client.timestamp_end = time.time()
await self.handle_hook(server_hooks.ClientDisconnectedHook(self.client)
)
if self.transports:
self.log("closing transports...", "debug")
for io in self.transports.values():
if io.handler:
asyncio_utils.cancel_task(io.handler,
"client disconnected")
await asyncio.wait(
[x.handler for x in self.transports.values() if x.handler])
self.log("transports closed!", "debug")
async def open_connection(self, command: commands.OpenConnection) -> None:
if not command.connection.address:
self.log(f"Cannot open connection, no hostname given.")
self.server_event(
events.OpenConnectionCompleted(
command, f"Cannot open connection, no hostname given."))
return
hook_data = server_hooks.ServerConnectionHookData(
client=self.client, server=command.connection)
await self.handle_hook(server_hooks.ServerConnectHook(hook_data))
if err := command.connection.error:
self.log(
f"server connection to {human.format_address(command.connection.address)} killed before connect: {err}"
)
self.server_event(
events.OpenConnectionCompleted(command,
f"Connection killed: {err}"))
return
async with self.max_conns[command.connection.address]:
try:
command.connection.timestamp_start = time.time()
reader, writer = await asyncio.open_connection(
*command.connection.address)
except (IOError, asyncio.CancelledError) as e:
err = str(e)
if not err: # str(CancelledError()) returns empty string.
err = "connection cancelled"
self.log(f"error establishing server connection: {err}")
command.connection.error = err
self.server_event(events.OpenConnectionCompleted(command, err))
if isinstance(e, asyncio.CancelledError):
# From https://docs.python.org/3/library/asyncio-exceptions.html#asyncio.CancelledError:
# > In almost all situations the exception must be re-raised.
# It is not really defined what almost means here, but we play safe.
raise
else:
command.connection.timestamp_tcp_setup = time.time()
command.connection.state = ConnectionState.OPEN
command.connection.peername = writer.get_extra_info('peername')
command.connection.sockname = writer.get_extra_info('sockname')
self.transports[command.connection].reader = reader
self.transports[command.connection].writer = writer
assert command.connection.peername
if command.connection.address[
0] != command.connection.peername[0]:
addr = f"{command.connection.address[0]} ({human.format_address(command.connection.peername)})"
else:
addr = human.format_address(command.connection.address)
self.log(f"server connect {addr}")
connected_hook = asyncio_utils.create_task(
self.handle_hook(
server_hooks.ServerConnectedHook(hook_data)),
name=f"handle_hook(server_connected) {addr}",
client=self.client.peername,
)
if not connected_hook:
return # this should not be needed, see asyncio_utils.create_task
self.server_event(events.OpenConnectionCompleted(
command, None))
# during connection opening, this function is the designated handler that can be cancelled.
# once we have a connection, we do want the teardown here to happen in any case, so we
# reassign the handler to .handle_connection and then clean up here once that is done.
new_handler = asyncio_utils.create_task(
self.handle_connection(command.connection),
name=f"server connection handler for {addr}",
client=self.client.peername,
)
if not new_handler:
return # this should not be needed, see asyncio_utils.create_task
self.transports[command.connection].handler = new_handler
await asyncio.wait([new_handler])
self.log(f"server disconnect {addr}")
command.connection.timestamp_end = time.time()
await connected_hook # wait here for this so that closed always comes after connected.
await self.handle_hook(
server_hooks.ServerDisconnectedHook(hook_data))
def flowdetails(state, flow: http.HTTPFlow):
text = []
sc = flow.server_conn
cc = flow.client_conn
req = flow.request
resp = flow.response
metadata = flow.metadata
if metadata is not None and len(metadata) > 0:
parts = [[str(k), repr(v)] for k, v in metadata.items()]
text.append(urwid.Text([("head", "Metadata:")]))
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4))
if sc is not None and sc.ip_address:
text.append(urwid.Text([("head", "Server Connection:")]))
parts = [
["Address", human.format_address(sc.address)],
]
if sc.ip_address:
parts.append(
["Resolved Address",
human.format_address(sc.ip_address)])
if resp:
parts.append(["HTTP Version", resp.http_version])
if sc.alpn_proto_negotiated:
parts.append(["ALPN", sc.alpn_proto_negotiated])
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4))
c = sc.cert
if c:
text.append(urwid.Text([("head", "Server Certificate:")]))
parts = [["Type", "%s, %s bits" % c.keyinfo],
["SHA1 digest", c.digest("sha1")],
["Valid to", str(c.notafter)],
["Valid from", str(c.notbefore)],
["Serial", str(c.serial)],
[
"Subject",
urwid.BoxAdapter(
urwid.ListBox(
common.format_keyvals(c.subject,
key="highlight",
val="text")),
len(c.subject))
],
[
"Issuer",
urwid.BoxAdapter(
urwid.ListBox(
common.format_keyvals(c.issuer,
key="highlight",
val="text")),
len(c.issuer))
]]
if c.altnames:
parts.append([
"Alt names", ", ".join(
strutils.bytes_to_escaped_str(x) for x in c.altnames)
])
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4))
if cc is not None:
text.append(urwid.Text([("head", "Client Connection:")]))
parts = [
["Address", "{}:{}".format(cc.address[0], cc.address[1])],
]
if req:
parts.append(["HTTP Version", req.http_version])
if cc.tls_version:
parts.append(["TLS Version", cc.tls_version])
if cc.sni:
parts.append(["Server Name Indication", cc.sni])
if cc.cipher_name:
parts.append(["Cipher Name", cc.cipher_name])
if cc.alpn_proto_negotiated:
parts.append(["ALPN", cc.alpn_proto_negotiated])
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4))
parts = []
if cc is not None and cc.timestamp_start:
parts.append([
"Client conn. established",
maybe_timestamp(cc, "timestamp_start")
])
if cc.ssl_established:
parts.append([
"Client conn. TLS handshake",
maybe_timestamp(cc, "timestamp_ssl_setup")
])
if sc is not None and sc.timestamp_start:
parts.append(
["Server conn. initiated",
maybe_timestamp(sc, "timestamp_start")])
parts.append([
"Server conn. TCP handshake",
maybe_timestamp(sc, "timestamp_tcp_setup")
])
if sc.ssl_established:
parts.append([
"Server conn. TLS handshake",
maybe_timestamp(sc, "timestamp_ssl_setup")
])
if req is not None and req.timestamp_start:
parts.append(
["First request byte",
maybe_timestamp(req, "timestamp_start")])
parts.append(
["Request complete",
maybe_timestamp(req, "timestamp_end")])
if resp is not None and resp.timestamp_start:
parts.append(
["First response byte",
maybe_timestamp(resp, "timestamp_start")])
parts.append(
["Response complete",
maybe_timestamp(resp, "timestamp_end")])
if parts:
# sort operations by timestamp
parts = sorted(parts, key=lambda p: p[1])
text.append(urwid.Text([("head", "Timing:")]))
text.extend(
common.format_keyvals(parts, key="key", val="text", indent=4))
return searchable.Searchable(text)
def log(self, msg, level):
msg = "{}: {}".format(human.format_address(self.client_conn.address),
msg)
self.channel.tell("log", log.LogEntry(msg, level))
def test_format_address():
assert human.format_address(("::1", "54010", "0", "0")) == "[::1]:54010"
assert human.format_address(("::ffff:127.0.0.1", "54010", "0", "0")) == "127.0.0.1:54010"
assert human.format_address(("127.0.0.1", "54010")) == "127.0.0.1:54010"
assert human.format_address(("example.com", "54010")) == "example.com:54010"