def receive_handshake_data(
self,
data: bytes) -> layer.CommandGenerator[Tuple[bool, Optional[str]]]:
if not self.send_connect:
return (yield from super().receive_handshake_data(data))
self.buf += data
response_head = self.buf.maybe_extract_lines()
if response_head:
response_head = [
bytes(x) for x in response_head
] # TODO: Make url.parse compatible with bytearrays
try:
response = http1_sansio.read_response_head(response_head)
except ValueError as e:
yield commands.Log(
f"{human.format_address(self.tunnel_connection.address)}: {e}"
)
return False, str(e)
if 200 <= response.status_code < 300:
if self.buf:
yield from self.receive_data(bytes(self.buf))
del self.buf
return True, None
else:
raw_resp = b"\n".join(response_head)
yield commands.Log(
f"{human.format_address(self.tunnel_connection.address)}: {raw_resp!r}",
level="debug")
return False, f"{response.status_code} {response.reason}"
else:
return False, None
def send_response(self, already_streamed: bool = False):
"""We have either consumed the entire response from the server or the response was set by an addon."""
assert self.flow.response
self.flow.response.timestamp_end = time.time()
yield HttpResponseHook(self.flow)
self.server_state = self.state_done
if (yield from self.check_killed(False)):
return
if not already_streamed:
content = self.flow.response.raw_content
yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response, not content), self.context.client)
if content:
yield SendHttp(ResponseData(self.stream_id, content), self.context.client)
yield SendHttp(ResponseEndOfMessage(self.stream_id), self.context.client)
if self.flow.response.status_code == 101:
is_websocket = (
self.flow.response.headers.get("upgrade", "").lower() == "websocket"
and
self.flow.request.headers.get("Sec-WebSocket-Version", "") == "13"
)
if is_websocket:
self.child_layer = websocket.WebsocketLayer(self.context, self.flow)
else:
self.child_layer = tcp.TCPLayer(self.context)
if self.debug:
yield commands.Log(f"{self.debug}[http] upgrading to {self.child_layer}", "debug")
yield from self.child_layer.handle_event(events.Start())
self._handle_event = self.passthrough
return
def _ask(self):
"""
Manually trigger a next_layer hook.
The only use at the moment is to make sure that the top layer is initialized.
"""
yield NextLayerHook(self)
# Has an addon decided on the next layer yet?
if self.layer:
if self.debug:
yield commands.Log(f"{self.debug}[nextlayer] {self.layer!r}",
"debug")
for e in self.events:
yield from self.layer.handle_event(e)
self.events.clear()
# Why do we need three assignments here?
# 1. When this function here is invoked we may have paused events. Those should be
# forwarded to the sublayer right away, so we reassign ._handle_event.
# 2. This layer is not needed anymore, so we directly reassign .handle_event.
# 3. Some layers may however still have a reference to the old .handle_event.
# ._handle is just an optimization to reduce the callstack in these cases.
self.handle_event = self.layer.handle_event
self._handle_event = self.layer._handle_event
self._handle = self.layer.handle_event
def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bool, Optional[str]]]:
if self.client_hello_parsed:
return (yield from super().receive_handshake_data(data))
self.recv_buffer.extend(data)
try:
client_hello = parse_client_hello(self.recv_buffer)
except ValueError:
return False, f"Cannot parse ClientHello: {self.recv_buffer.hex()}"
if client_hello:
self.client_hello_parsed = True
else:
return False, None
self.conn.sni = client_hello.sni
self.conn.alpn_offers = client_hello.alpn_protocols
tls_clienthello = ClientHelloData(self.context)
yield TlsClienthelloHook(tls_clienthello)
if tls_clienthello.establish_server_tls_first and not self.context.server.tls_established:
err = yield from self.start_server_tls()
if err:
yield commands.Log(f"Unable to establish TLS connection with server ({err}). "
f"Trying to establish TLS with client anyway.")
yield from self.start_tls()
ret = yield from super().receive_handshake_data(bytes(self.recv_buffer))
self.recv_buffer.clear()
return ret
def receive_data(self, data: bytes) -> layer.CommandGenerator[None]:
if data:
self.tls.bio_write(data)
yield from self.tls_interact()
plaintext = bytearray()
close = False
while True:
try:
plaintext.extend(self.tls.recv(65535))
except SSL.WantReadError:
break
except SSL.ZeroReturnError:
close = True
break
if plaintext:
yield from self.event_to_child(
events.DataReceived(self.conn, bytes(plaintext))
)
if close:
self.conn.state &= ~context.ConnectionState.CAN_READ
if self.debug:
yield commands.Log(f"{self.debug}[tls] close_notify {self.conn}", level="debug")
yield from self.event_to_child(
events.ConnectionClosed(self.conn)
)
def read_headers(
self,
event: events.ConnectionEvent) -> layer.CommandGenerator[None]:
if isinstance(event, events.DataReceived):
if not self.request:
# we just received some data for an unknown request.
yield commands.Log(
f"Unexpected data from server: {bytes(self.buf)!r}")
yield commands.CloseConnection(self.conn)
return
assert self.stream_id
response_head = self.buf.maybe_extract_lines()
if response_head:
response_head = [
bytes(x) for x in response_head
] # TODO: Make url.parse compatible with bytearrays
try:
self.response = http1_sansio.read_response_head(
response_head)
expected_size = http1_sansio.expected_http_body_size(
self.request, self.response)
except (ValueError, exceptions.HttpSyntaxException) as e:
yield commands.CloseConnection(self.conn)
yield ReceiveHttp(
ResponseProtocolError(
self.stream_id,
f"Cannot parse HTTP response: {e}"))
return
yield ReceiveHttp(
ResponseHeaders(self.stream_id, self.response,
expected_size == 0))
self.body_reader = make_body_reader(expected_size)
self.state = self.read_body
yield from self.state(event)
else:
pass # FIXME: protect against header size DoS
elif isinstance(event, events.ConnectionClosed):
if self.conn.state & ConnectionState.CAN_WRITE:
yield commands.CloseConnection(self.conn)
if self.stream_id:
if self.buf:
yield ReceiveHttp(
ResponseProtocolError(
self.stream_id,
f"unexpected server response: {bytes(self.buf)!r}")
)
else:
# The server has closed the connection to prevent us from continuing.
# We need to signal that to the stream.
# https://tools.ietf.org/html/rfc7231#section-6.5.11
yield ReceiveHttp(
ResponseProtocolError(self.stream_id,
"server closed connection"))
else:
return
else:
raise AssertionError(f"Unexpected event: {event}")
def __debug(self, message):
if len(message) > 512:
message = message[:512] + "…"
if Layer.__last_debug_message == message:
message = message.split("\n", 1)[0].strip()
if len(message) > 256:
message = message[:256] + "…"
else:
Layer.__last_debug_message = message
return commands.Log(textwrap.indent(message, self.debug), "debug")
def _handle_event(self,
event: events.Event) -> layer.CommandGenerator[None]:
assert platform.original_addr is not None
socket = yield commands.GetSocket(self.context.client)
try:
self.context.server.address = platform.original_addr(socket)
except Exception as e:
yield commands.Log(f"Transparent mode failure: {e!r}")
self.child_layer = layer.NextLayer(self.context)
yield from self.finish_start()
def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:
if self.conn.sni:
assert isinstance(self.conn.sni, bytes)
dest = self.conn.sni.decode("idna")
else:
dest = human.format_address(self.context.server.address)
if err.startswith("Cannot parse ClientHello"):
pass
elif "unknown ca" in err or "bad certificate" in err:
err = f"The client does not trust the proxy's certificate for {dest} ({err})"
else:
err = f"The client may not trust the proxy's certificate for {dest} ({err})"
yield commands.Log(f"Client TLS handshake failed. {err}", level="warn")
yield from super().on_handshake_error(err)
def receive_handshake_data(self, data: bytes) -> layer.CommandGenerator[Tuple[bool, Optional[str]]]:
# bio_write errors for b"", so we need to check first if we actually received something.
if data:
self.tls.bio_write(data)
try:
self.tls.do_handshake()
except SSL.WantReadError:
yield from self.tls_interact()
return False, None
except SSL.Error as e:
# provide more detailed information for some errors.
last_err = e.args and isinstance(e.args[0], list) and e.args[0] and e.args[0][-1]
if last_err == ('SSL routines', 'tls_process_server_certificate', 'certificate verify failed'):
verify_result = SSL._lib.SSL_get_verify_result(self.tls._ssl)
error = SSL._ffi.string(SSL._lib.X509_verify_cert_error_string(verify_result)).decode()
err = f"Certificate verify failed: {error}"
elif last_err in [
('SSL routines', 'ssl3_read_bytes', 'tlsv1 alert unknown ca'),
('SSL routines', 'ssl3_read_bytes', 'sslv3 alert bad certificate')
]:
assert isinstance(last_err, tuple)
err = last_err[2]
elif last_err == ('SSL routines', 'ssl3_get_record', 'wrong version number') and data[:4].isascii():
err = f"The remote server does not speak TLS."
else: # pragma: no cover
# TODO: Add test case one we find one.
err = f"OpenSSL {e!r}"
return False, err
else:
# Get all peer certificates.
# https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_cert_chain.html
# If called on the client side, the stack also contains the peer's certificate; if called on the server
# side, the peer's certificate must be obtained separately using SSL_get_peer_certificate(3).
all_certs = self.tls.get_peer_cert_chain() or []
if self.conn == self.context.client:
cert = self.tls.get_peer_certificate()
if cert:
all_certs.insert(0, cert)
self.conn.timestamp_tls_setup = time.time()
self.conn.sni = self.tls.get_servername()
self.conn.alpn = self.tls.get_alpn_proto_negotiated()
self.conn.certificate_list = [certs.Cert(x) for x in all_certs]
self.conn.cipher = self.tls.get_cipher_name()
self.conn.cipher_list = self.tls.get_cipher_list()
self.conn.tls_version = self.tls.get_protocol_version_name()
if self.debug:
yield commands.Log(f"{self.debug}[tls] tls established: {self.conn}", "debug")
yield from self.receive_data(b"")
return True, None
def read_headers(
self,
event: events.ConnectionEvent) -> layer.CommandGenerator[None]:
if isinstance(event, events.DataReceived):
request_head = self.buf.maybe_extract_lines()
if request_head:
request_head = [
bytes(x) for x in request_head
] # TODO: Make url.parse compatible with bytearrays
try:
self.request = http1_sansio.read_request_head(request_head)
expected_body_size = http1_sansio.expected_http_body_size(
self.request, expect_continue_as_0=False)
except (ValueError, exceptions.HttpSyntaxException) as e:
yield commands.Log(
f"{human.format_address(self.conn.peername)}: {e}")
yield commands.CloseConnection(self.conn)
self.state = self.done
return
yield ReceiveHttp(
RequestHeaders(self.stream_id, self.request,
expected_body_size == 0))
self.body_reader = make_body_reader(expected_body_size)
self.state = self.read_body
yield from self.state(event)
else:
pass # FIXME: protect against header size DoS
elif isinstance(event, events.ConnectionClosed):
buf = bytes(self.buf)
if buf.strip():
yield commands.Log(
f"Client closed connection before completing request headers: {buf!r}"
)
yield commands.CloseConnection(self.conn)
else:
raise AssertionError(f"Unexpected event: {event}")
def start(self, _) -> layer.CommandGenerator[None]:
client_extensions = []
server_extensions = []
# Parse extension headers. We only support deflate at the moment and ignore everything else.
ext_header = self.flow.handshake_flow.response.headers.get(
"Sec-WebSocket-Extensions", "")
if ext_header:
for ext in wsproto.utilities.split_comma_header(
ext_header.encode("ascii", "replace")):
ext_name = ext.split(";", 1)[0].strip()
if ext_name == wsproto.extensions.PerMessageDeflate.name:
client_deflate = wsproto.extensions.PerMessageDeflate()
client_deflate.finalize(ext)
client_extensions.append(client_deflate)
server_deflate = wsproto.extensions.PerMessageDeflate()
server_deflate.finalize(ext)
server_extensions.append(server_deflate)
else:
yield commands.Log(
f"Ignoring unknown WebSocket extension {ext_name!r}.")
self.client_ws = WebsocketConnection(wsproto.ConnectionType.SERVER,
client_extensions,
conn=self.context.client)
self.server_ws = WebsocketConnection(wsproto.ConnectionType.CLIENT,
server_extensions,
conn=self.context.server)
yield WebsocketStartHook(self.flow)
if self.flow.stream: # pragma: no cover
raise NotImplementedError(
"WebSocket streaming is not supported at the moment.")
self._handle_event = self.relay_messages
def test_dataclasses(tconn):
assert repr(commands.SendData(tconn, b"foo"))
assert repr(commands.OpenConnection(tconn))
assert repr(commands.CloseConnection(tconn))
assert repr(commands.GetSocket(tconn))
assert repr(commands.Log("hello", "info"))
def relay_messages(
self,
event: events.ConnectionEvent) -> layer.CommandGenerator[None]:
from_client = event.connection == self.context.client
from_str = 'client' if from_client else 'server'
if from_client:
src_ws = self.client_ws
dst_ws = self.server_ws
else:
src_ws = self.server_ws
dst_ws = self.client_ws
if isinstance(event, events.DataReceived):
src_ws.receive_data(event.data)
elif isinstance(event, events.ConnectionClosed):
src_ws.receive_data(None)
else: # pragma: no cover
raise AssertionError(f"Unexpected event: {event}")
for ws_event in src_ws.events():
if isinstance(ws_event, wsproto.events.Message):
src_ws.frame_buf.append(ws_event.data)
if ws_event.message_finished:
if isinstance(ws_event, wsproto.events.TextMessage):
frame_type = Opcode.TEXT
content = "".join(src_ws.frame_buf) # type: ignore
else:
frame_type = Opcode.BINARY
content = b"".join(src_ws.frame_buf) # type: ignore
fragmentizer = Fragmentizer(src_ws.frame_buf)
src_ws.frame_buf.clear()
message = websocket.WebSocketMessage(
frame_type, from_client, content)
self.flow.messages.append(message)
yield WebsocketMessageHook(self.flow)
assert not message.killed # this is deprecated, instead we should have .content set to emptystr.
for msg in fragmentizer(message.content):
yield dst_ws.send2(msg)
elif isinstance(ws_event,
(wsproto.events.Ping, wsproto.events.Pong)):
yield commands.Log(
f"Received WebSocket {ws_event.__class__.__name__.lower()} from {from_str} "
f"(payload: {bytes(ws_event.payload)!r})")
yield dst_ws.send2(ws_event)
elif isinstance(ws_event, wsproto.events.CloseConnection):
self.flow.close_sender = from_str
self.flow.close_code = ws_event.code
self.flow.close_reason = ws_event.reason
for ws in [self.server_ws, self.client_ws]:
if ws.state in {
ConnectionState.OPEN,
ConnectionState.REMOTE_CLOSING
}:
# response == original event, so no need to differentiate here.
yield ws.send2(ws_event)
yield commands.CloseConnection(ws.conn)
if ws_event.code in {1000, 1001, 1005}:
yield WebsocketEndHook(self.flow)
else:
self.flow.error = flow.Error(
f"WebSocket Error: {format_close_event(ws_event)}")
yield WebsocketErrorHook(self.flow)
self._handle_event = self.done
else: # pragma: no cover
raise AssertionError(f"Unexpected WebSocket event: {ws_event}")
def on_handshake_error(self, err: str) -> layer.CommandGenerator[None]:
yield commands.Log(f"Server TLS handshake failed. {err}", level="warn")
yield from super().on_handshake_error(err)