def open_h2_server_conn():
# this is a bit fake here (port 80, with alpn, but no tls - c'mon),
# but we don't want to pollute our tests with TLS handshakes.
s = Server(("example.com", 80))
s.state = ConnectionState.OPEN
s.alpn = b"h2"
return s
def test_tunnel_handshake_start(tctx: Context, success):
server = Server(("proxy", 1234))
server.state = ConnectionState.OPEN
tl = TTunnelLayer(tctx, server, tctx.server)
tl.child_layer = TChildLayer(tctx)
assert repr(tl)
playbook = Playbook(tl, logs=True)
(playbook << SendData(server, b"handshake-hello") >> DataReceived(
tctx.client, b"client-hello") >> DataReceived(
server, b"handshake-" + success.encode()) << SendData(
server, b"handshake-" + success.encode()))
if success == "success":
playbook << Log("Got start. Server state: OPEN")
else:
playbook << CloseConnection(server)
playbook << Log("Got start. Server state: CLOSED")
playbook << SendData(tctx.client, b"client-hello-reply")
if success == "success":
playbook >> DataReceived(server, b"tunneled-server-hello")
playbook << SendData(server, b"tunneled-server-hello-reply")
assert playbook
def get_connection(self, event: GetHttpConnection, *, reuse: bool = True) -> layer.CommandGenerator[None]:
# Do we already have a connection we can re-use?
if reuse:
for connection in self.connections:
# see "tricky multiplexing edge case" in make_http_connection for an explanation
conn_is_pending_or_h2 = (
connection.alpn == b"h2"
or connection in self.waiting_for_establishment
)
h2_to_h1 = self.context.client.alpn == b"h2" and not conn_is_pending_or_h2
connection_suitable = (
event.connection_spec_matches(connection)
and not h2_to_h1
)
if connection_suitable:
if connection in self.waiting_for_establishment:
self.waiting_for_establishment[connection].append(event)
return
elif connection.connected:
stream = self.command_sources.pop(event)
yield from self.event_to_child(stream, GetHttpConnectionCompleted(event, (connection, None)))
return
else:
pass # the connection is at least half-closed already, we want a new one.
can_use_context_connection = (
self.context.server not in self.connections and
self.context.server.connected and
event.connection_spec_matches(self.context.server)
)
context = self.context.fork()
stack = tunnel.LayerStack()
if not can_use_context_connection:
context.server = Server(event.address)
if event.tls:
context.server.sni = event.address[0]
if event.via:
assert event.via.scheme in ("http", "https")
http_proxy = Server(event.via.address)
if event.via.scheme == "https":
http_proxy.alpn_offers = tls.HTTP_ALPNS
http_proxy.sni = event.via.address[0]
stack /= tls.ServerTLSLayer(context, http_proxy)
send_connect = not (self.mode == HTTPMode.upstream and not event.tls)
stack /= _upstream_proxy.HttpUpstreamProxy(context, http_proxy, send_connect)
if event.tls:
stack /= tls.ServerTLSLayer(context)
stack /= HttpClient(context)
self.connections[context.server] = stack[0]
self.waiting_for_establishment[context.server].append(event)
yield from self.event_to_child(stack[0], events.Start())
def test_disconnect_during_handshake_start(tctx: Context, disconnect):
server = Server(("proxy", 1234))
server.state = ConnectionState.OPEN
tl = TTunnelLayer(tctx, server, tctx.server)
tl.child_layer = TChildLayer(tctx)
playbook = Playbook(tl, logs=True)
assert (
playbook
<< SendData(server, b"handshake-hello")
)
if disconnect == "client":
assert (
playbook
>> ConnectionClosed(tctx.client)
>> ConnectionClosed(server) # proxyserver will cancel all other connections as well.
<< CloseConnection(server)
<< Log("Got start. Server state: CLOSED")
<< Log("Got client close.")
<< CloseConnection(tctx.client)
)
else:
assert (
playbook
>> ConnectionClosed(server)
<< CloseConnection(server)
<< Log("Got start. Server state: CLOSED")
)
def test_tunnel_default_impls(tctx: Context):
"""
Some tunnels don't need certain features, so the default behaviour
should be to be transparent.
"""
server = Server(None)
server.state = ConnectionState.OPEN
tl = tunnel.TunnelLayer(tctx, server, tctx.server)
tl.child_layer = TChildLayer(tctx)
playbook = Playbook(tl, logs=True)
assert (
playbook
<< Log("Got start. Server state: OPEN")
>> DataReceived(server, b"server-hello")
<< SendData(server, b"server-hello-reply")
)
assert tl.tunnel_state is tunnel.TunnelState.OPEN
assert (
playbook
>> ConnectionClosed(server)
<< Log("Got server close.")
<< CloseConnection(server)
)
assert tl.tunnel_state is tunnel.TunnelState.CLOSED
assert (
playbook
>> DataReceived(tctx.client, b"open")
<< OpenConnection(server)
>> reply(None)
<< Log("Opened: err=None. Server state: OPEN")
>> DataReceived(server, b"half-close")
<< CloseConnection(server, half_close=True)
)
def test_basic(self):
s = Server(("address", 22))
assert repr(s)
assert str(s)
s.timestamp_tls_setup = 1607780791
assert str(s)
s.alpn = b"foo"
s.sockname = ("127.0.0.1", 54321)
assert str(
s) == "Server(address:22, state=closed, alpn=foo, src_port=54321)"
def handle_connect_upstream(self):
assert self.context.server.via.scheme in ("http", "https")
http_proxy = Server(self.context.server.via.address)
stack = tunnel.LayerStack()
if self.context.server.via.scheme == "https":
http_proxy.sni = self.context.server.via.address[0]
stack /= tls.ServerTLSLayer(self.context, http_proxy)
stack /= _upstream_proxy.HttpUpstreamProxy(self.context, http_proxy, True)
self.child_layer = stack[0]
yield from self.handle_connect_finish()
def test_cannot_parse_clienthello(self, tctx: context.Context):
"""Test the scenario where we cannot parse the ClientHello"""
playbook, client_layer, tssl_client = make_client_tls_layer(tctx)
tls_hook_data = tutils.Placeholder(TlsData)
invalid = b"\x16\x03\x01\x00\x00"
assert (playbook >> events.DataReceived(
tctx.client, invalid
) << commands.Log(
f"Client TLS handshake failed. Cannot parse ClientHello: {invalid.hex()}",
level="warn") << tls.TlsFailedClientHook(tls_hook_data) >>
tutils.reply() << commands.CloseConnection(tctx.client))
assert tls_hook_data().conn.error
assert not tctx.client.tls_established
# Make sure that an active server connection does not cause child layers to spawn.
client_layer.debug = ""
assert (playbook >> events.DataReceived(
Server(None), b"data on other stream"
) << commands.Log(
">> DataReceived(server, b'data on other stream')", 'debug'
) << commands.Log(
"Swallowing DataReceived(server, b'data on other stream') as handshake failed.",
"debug"))
def test_disconnect_during_handshake_command(tctx: Context, disconnect):
server = Server(("proxy", 1234))
tl = TTunnelLayer(tctx, server, tctx.server)
tl.child_layer = TChildLayer(tctx)
playbook = Playbook(tl, logs=True)
assert (
playbook
<< Log("Got start. Server state: CLOSED")
>> DataReceived(tctx.client, b"client-hello")
<< SendData(tctx.client, b"client-hello-reply")
>> DataReceived(tctx.client, b"open")
<< OpenConnection(server)
>> reply(None)
<< SendData(server, b"handshake-hello")
)
if disconnect == "client":
assert (
playbook
>> ConnectionClosed(tctx.client)
>> ConnectionClosed(server) # proxyserver will cancel all other connections as well.
<< CloseConnection(server)
<< Log("Opened: err='connection closed without notice'. Server state: CLOSED")
<< Log("Got client close.")
<< CloseConnection(tctx.client)
)
else:
assert (
playbook
>> ConnectionClosed(server)
<< CloseConnection(server)
<< Log("Opened: err='connection closed without notice'. Server state: CLOSED")
)
def test_tunnel_handshake_command(tctx: Context, success):
server = Server(("proxy", 1234))
tl = TTunnelLayer(tctx, server, tctx.server)
tl.child_layer = TChildLayer(tctx)
playbook = Playbook(tl, logs=True)
(playbook << Log("Got start. Server state: CLOSED") >> DataReceived(
tctx.client, b"client-hello") << SendData(
tctx.client, b"client-hello-reply") >> DataReceived(
tctx.client, b"open") << OpenConnection(server) >> reply(None)
<< SendData(server, b"handshake-hello") >> DataReceived(
server, b"handshake-" + success.encode()) << SendData(
server, b"handshake-" + success.encode()))
if success == "success":
assert (playbook << Log(f"Opened: err=None. Server state: OPEN") >>
DataReceived(server, b"tunneled-server-hello") << SendData(
server, b"tunneled-server-hello-reply") >>
ConnectionClosed(tctx.client) << Log("Got client close.") <<
CloseConnection(tctx.client))
assert tl.tunnel_state is tunnel.TunnelState.OPEN
assert (playbook >> ConnectionClosed(server) <<
Log("Got server close.") << CloseConnection(server))
assert tl.tunnel_state is tunnel.TunnelState.CLOSED
else:
assert (playbook << CloseConnection(server) <<
Log("Opened: err='handshake error'. Server state: CLOSED"))
assert tl.tunnel_state is tunnel.TunnelState.CLOSED
def test_client_only(self, tctx: context.Context):
"""Test TLS with client only"""
playbook, client_layer, tssl_client = make_client_tls_layer(tctx)
client_layer.debug = " "
assert not tctx.client.tls_established
# Send ClientHello, receive ServerHello
data = tutils.Placeholder(bytes)
assert (playbook >> events.DataReceived(
tctx.client, tssl_client.bio_read()) << tls.TlsClienthelloHook(
tutils.Placeholder()) >> tutils.reply() << tls.TlsStartHook(
tutils.Placeholder()) >> reply_tls_start() <<
commands.SendData(tctx.client, data))
tssl_client.bio_write(data())
tssl_client.do_handshake()
# Finish Handshake
interact(playbook, tctx.client, tssl_client)
assert tssl_client.obj.getpeercert(True)
assert tctx.client.tls_established
# Echo
_test_echo(playbook, tssl_client, tctx.client)
other_server = Server(None)
assert (playbook >> events.DataReceived(other_server, b"Plaintext") <<
commands.SendData(other_server, b"plaintext"))
def request(flow: http.HTTPFlow) -> None:
address = proxy_address(flow)
is_proxy_change = address != flow.server_conn.via.address
server_connection_already_open = flow.server_conn.timestamp_start is not None
if is_proxy_change and server_connection_already_open:
# server_conn already refers to an existing connection (which cannot be modified),
# so we need to replace it with a new server connection object.
flow.server_conn = Server(flow.server_conn.address)
flow.server_conn.via = ServerSpec("http", address)
def test_tunnel_openconnection_error(tctx: Context):
server = Server(("proxy", 1234))
tl = TTunnelLayer(tctx, server, tctx.server)
tl.child_layer = TChildLayer(tctx)
playbook = Playbook(tl, logs=True)
assert (playbook << Log("Got start. Server state: CLOSED") >> DataReceived(
tctx.client, b"open") << OpenConnection(server))
assert tl.tunnel_state is tunnel.TunnelState.ESTABLISHING
assert (playbook >> reply("IPoAC packet dropped.") <<
Log("Opened: err='IPoAC packet dropped.'. Server state: CLOSED"))
assert tl.tunnel_state is tunnel.TunnelState.CLOSED
def test_address(self):
s = Server(("address", 22))
s.address = ("example.com", 443)
s.state = ConnectionState.OPEN
with pytest.raises(RuntimeError):
s.address = ("example.com", 80)
# No-op assignment, allowed because it might be triggered by a Server.set_state() call.
s.address = ("example.com", 443)
def __init__(self, flow: http.HTTPFlow, options: Options) -> None:
client = flow.client_conn.copy()
client.state = ConnectionState.OPEN
context = Context(client, options)
context.server = Server((flow.request.host, flow.request.port))
context.server.tls = flow.request.scheme == "https"
if options.mode.startswith("upstream:"):
context.server.via = server_spec.parse_with_mode(options.mode)[1]
super().__init__(context)
self.layer = layers.HttpLayer(context, HTTPMode.transparent)
self.layer.connections[client] = MockServer(flow, context.fork())
self.flow = flow
self.done = asyncio.Event()
def get_connection(self,
event: GetHttpConnection,
*,
reuse: bool = True) -> layer.CommandGenerator[None]:
# Do we already have a connection we can re-use?
if reuse:
for connection in self.connections:
connection_suitable = (
event.connection_spec_matches(connection))
if connection_suitable:
if connection in self.waiting_for_establishment:
self.waiting_for_establishment[connection].append(
event)
return
elif connection.error:
stream = self.command_sources.pop(event)
yield from self.event_to_child(
stream,
GetHttpConnectionCompleted(
event, (None, connection.error)))
return
elif connection.connected:
# see "tricky multiplexing edge case" in make_http_connection for an explanation
h2_to_h1 = self.context.client.alpn == b"h2" and connection.alpn != b"h2"
if not h2_to_h1:
stream = self.command_sources.pop(event)
yield from self.event_to_child(
stream,
GetHttpConnectionCompleted(
event, (connection, None)))
return
else:
pass # the connection is at least half-closed already, we want a new one.
context_connection_matches = (
self.context.server not in self.connections
and event.connection_spec_matches(self.context.server))
can_use_context_connection = (context_connection_matches
and self.context.server.connected)
if context_connection_matches and self.context.server.error:
stream = self.command_sources.pop(event)
yield from self.event_to_child(
stream,
GetHttpConnectionCompleted(event,
(None, self.context.server.error)))
return
context = self.context.fork()
stack = tunnel.LayerStack()
if not can_use_context_connection:
context.server = Server(event.address)
if event.via:
context.server.via = event.via
assert event.via.scheme in ("http", "https")
# We always send a CONNECT request, *except* for plaintext absolute-form HTTP requests in upstream mode.
send_connect = event.tls or self.mode != HTTPMode.upstream
stack /= _upstream_proxy.HttpUpstreamProxy.make(
context, send_connect)
if event.tls:
# Assume that we are in transparent mode and lazily did not open a connection yet.
# We don't want the IP (which is the address) as the upstream SNI, but the client's SNI instead.
if self.mode == HTTPMode.transparent and event.address == self.context.server.address:
context.server.sni = self.context.client.sni or event.address[
0]
else:
context.server.sni = event.address[0]
stack /= tls.ServerTLSLayer(context)
stack /= HttpClient(context)
self.connections[context.server] = stack[0]
self.waiting_for_establishment[context.server].append(event)
yield from self.event_to_child(stack[0], events.Start())
def test_address(self):
s = Server(("address", 22))
s.address = ("example.com", 443)
s.state = ConnectionState.OPEN
with pytest.raises(RuntimeError):
s.address = ("example.com", 80)