def handle_connect_finish(self):
if not self.flow.response:
# Do not send any response headers as it breaks proxying non-80 ports on
# Android emulators using the -http-proxy option.
self.flow.response = http.Response(
self.flow.request.data.http_version,
200,
b"Connection established",
http.Headers(),
b"",
None,
time.time(),
time.time(),
)
if 200 <= self.flow.response.status_code < 300:
self.child_layer = self.child_layer or layer.NextLayer(
self.context)
yield from self.child_layer.handle_event(events.Start())
self._handle_event = self.passthrough
yield SendHttp(
ResponseHeaders(self.stream_id, self.flow.response, True),
self.context.client)
yield SendHttp(ResponseEndOfMessage(self.stream_id),
self.context.client)
else:
yield from self.send_response()
def get_connection(self, event: GetHttpConnection, *, reuse: bool = True) -> layer.CommandGenerator[None]:
# Do we already have a connection we can re-use?
if reuse:
for connection in self.connections:
# see "tricky multiplexing edge case" in make_http_connection for an explanation
conn_is_pending_or_h2 = (
connection.alpn == b"h2"
or connection in self.waiting_for_establishment
)
h2_to_h1 = self.context.client.alpn == b"h2" and not conn_is_pending_or_h2
connection_suitable = (
event.connection_spec_matches(connection)
and not h2_to_h1
)
if connection_suitable:
if connection in self.waiting_for_establishment:
self.waiting_for_establishment[connection].append(event)
return
elif connection.connected:
stream = self.command_sources.pop(event)
yield from self.event_to_child(stream, GetHttpConnectionCompleted(event, (connection, None)))
return
else:
pass # the connection is at least half-closed already, we want a new one.
can_use_context_connection = (
self.context.server not in self.connections and
self.context.server.connected and
event.connection_spec_matches(self.context.server)
)
context = self.context.fork()
stack = tunnel.LayerStack()
if not can_use_context_connection:
context.server = Server(event.address)
if event.tls:
context.server.sni = event.address[0]
if event.via:
assert event.via.scheme in ("http", "https")
http_proxy = Server(event.via.address)
if event.via.scheme == "https":
http_proxy.alpn_offers = tls.HTTP_ALPNS
http_proxy.sni = event.via.address[0]
stack /= tls.ServerTLSLayer(context, http_proxy)
send_connect = not (self.mode == HTTPMode.upstream and not event.tls)
stack /= _upstream_proxy.HttpUpstreamProxy(context, http_proxy, send_connect)
if event.tls:
stack /= tls.ServerTLSLayer(context)
stack /= HttpClient(context)
self.connections[context.server] = stack[0]
self.waiting_for_establishment[context.server].append(event)
yield from self.event_to_child(stack[0], events.Start())
def send_response(self, already_streamed: bool = False):
"""We have either consumed the entire response from the server or the response was set by an addon."""
assert self.flow.response
self.flow.response.timestamp_end = time.time()
yield HttpResponseHook(self.flow)
self.server_state = self.state_done
if (yield from self.check_killed(False)):
return
if not already_streamed:
content = self.flow.response.raw_content
yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response, not content), self.context.client)
if content:
yield SendHttp(ResponseData(self.stream_id, content), self.context.client)
yield SendHttp(ResponseEndOfMessage(self.stream_id), self.context.client)
if self.flow.response.status_code == 101:
is_websocket = (
self.flow.response.headers.get("upgrade", "").lower() == "websocket"
and
self.flow.request.headers.get("Sec-WebSocket-Version", "") == "13"
)
if is_websocket:
self.child_layer = websocket.WebsocketLayer(self.context, self.flow)
else:
self.child_layer = tcp.TCPLayer(self.context)
if self.debug:
yield commands.Log(f"{self.debug}[http] upgrading to {self.child_layer}", "debug")
yield from self.child_layer.handle_event(events.Start())
self._handle_event = self.passthrough
return
def send_response(self, already_streamed: bool = False):
"""We have either consumed the entire response from the server or the response was set by an addon."""
assert self.flow.response
self.flow.response.timestamp_end = time.time()
is_websocket = (self.flow.response.status_code == 101
and self.flow.response.headers.get(
"upgrade", "").lower() == "websocket"
and self.flow.request.headers.get(
"Sec-WebSocket-Version",
"").encode() == wsproto.handshake.WEBSOCKET_VERSION
and self.context.options.websocket)
if is_websocket:
# We need to set this before calling the response hook
# so that addons can determine if a WebSocket connection is following up.
self.flow.websocket = WebSocketData()
yield HttpResponseHook(self.flow)
self.server_state = self.state_done
if (yield from self.check_killed(False)):
return
if not already_streamed:
content = self.flow.response.raw_content
done_after_headers = not (content or self.flow.response.trailers)
yield SendHttp(
ResponseHeaders(self.stream_id, self.flow.response,
done_after_headers), self.context.client)
if content:
yield SendHttp(ResponseData(self.stream_id, content),
self.context.client)
if self.flow.response.trailers:
yield SendHttp(
ResponseTrailers(self.stream_id, self.flow.response.trailers),
self.context.client)
yield SendHttp(ResponseEndOfMessage(self.stream_id),
self.context.client)
if self.flow.response.status_code == 101:
if is_websocket:
self.child_layer = websocket.WebsocketLayer(
self.context, self.flow)
elif self.context.options.rawtcp:
self.child_layer = tcp.TCPLayer(self.context)
else:
yield commands.Log(
f"Sent HTTP 101 response, but no protocol is enabled to upgrade to.",
"warn")
yield commands.CloseConnection(self.context.client)
self.client_state = self.server_state = self.state_errored
return
if self.debug:
yield commands.Log(
f"{self.debug}[http] upgrading to {self.child_layer}",
"debug")
yield from self.child_layer.handle_event(events.Start())
self._handle_event = self.passthrough
return
def finish_start(self) -> layer.CommandGenerator[Optional[str]]:
if self.context.options.connection_strategy == "eager":
err = yield commands.OpenConnection(self.context.server)
if err:
self._handle_event = self.done # type: ignore
return err
self._handle_event = self.child_layer.handle_event # type: ignore
yield from self.child_layer.handle_event(events.Start())
return None
def finish_start(self):
if self.context.options.connection_strategy == "eager":
err = yield commands.OpenConnection(self.context.server)
if err:
yield commands.CloseConnection(self.context.client)
self._handle_event = self.done
return
self._handle_event = self.child_layer.handle_event
yield from self.child_layer.handle_event(events.Start())
def _handle_event(self, event: events.Event):
if isinstance(event, events.Start):
yield from self.event_to_child(
self.connections[self.context.client], event)
if self.mode is HTTPMode.upstream:
self.context.server.via = server_spec.parse_with_mode(
self.context.options.mode)[1]
elif isinstance(event, events.CommandCompleted):
stream = self.command_sources.pop(event.command)
yield from self.event_to_child(stream, event)
elif isinstance(event, events.MessageInjected):
# For injected messages we pass the HTTP stacks entirely and directly address the stream.
try:
conn = self.connections[event.flow.server_conn]
except KeyError:
# We have a miss for the server connection, which means we're looking at a connection object
# that is tunneled over another connection (for example: over an upstream HTTP proxy).
# We now take the stream associated with the client connection. That won't work for HTTP/2,
# but it's good enough for HTTP/1.
conn = self.connections[event.flow.client_conn]
if isinstance(conn, HttpStream):
stream_id = conn.stream_id
else:
# We reach to the end of the connection's child stack to get the HTTP/1 client layer,
# which tells us which stream we are dealing with.
conn = conn.context.layers[-1]
assert isinstance(conn, Http1Connection)
assert conn.stream_id
stream_id = conn.stream_id
yield from self.event_to_child(self.streams[stream_id], event)
elif isinstance(event, events.ConnectionEvent):
if event.connection == self.context.server and self.context.server not in self.connections:
# We didn't do anything with this connection yet, now the peer is doing something.
if isinstance(event, events.ConnectionClosed):
# The peer has closed it - let's close it too!
yield commands.CloseConnection(event.connection)
elif isinstance(event, events.DataReceived):
# The peer has sent data. This can happen with HTTP/2 servers that already send a settings frame.
child_layer: HttpConnection
if self.context.server.alpn == b"h2":
child_layer = Http2Client(self.context.fork())
else:
child_layer = Http1Client(self.context.fork())
self.connections[self.context.server] = child_layer
yield from self.event_to_child(child_layer, events.Start())
yield from self.event_to_child(child_layer, event)
else:
raise AssertionError(f"Unexpected event: {event}")
else:
handler = self.connections[event.connection]
yield from self.event_to_child(handler, event)
else:
raise AssertionError(f"Unexpected event: {event}")
def handle_connect_finish(self):
if not self.flow.response:
self.flow.response = http.make_connect_response(self.flow.request.data.http_version)
if 200 <= self.flow.response.status_code < 300:
yield SendHttp(ResponseHeaders(self.stream_id, self.flow.response), self.context.client)
yield SendHttp(ResponseEndOfMessage(self.stream_id), self.context.client)
self.child_layer = self.child_layer or layer.NextLayer(self.context)
yield from self.child_layer.handle_event(events.Start())
self._handle_event = self.passthrough
else:
yield from self.send_response()
def __init__(
self,
layer: Layer,
hooks: bool = True,
logs: bool = False,
expected: typing.Optional[PlaybookEntryList] = None,
):
if expected is None:
expected = [events.Start()]
self.layer = layer
self.expected = expected
self.actual = []
self._errored = False
self.logs = logs
self.hooks = hooks
async def handle_client(self) -> None:
watch = asyncio_utils.create_task(
self.timeout_watchdog.watch(),
name="timeout watchdog",
client=self.client.peername,
)
if not watch:
return # this should not be needed, see asyncio_utils.create_task
self.log("client connect")
await self.handle_hook(server_hooks.ClientConnectedHook(self.client))
if self.client.error:
self.log("client kill connection")
writer = self.transports.pop(self.client).writer
assert writer
writer.close()
else:
handler = asyncio_utils.create_task(
self.handle_connection(self.client),
name=f"client connection handler",
client=self.client.peername,
)
if not handler:
return # this should not be needed, see asyncio_utils.create_task
self.transports[self.client].handler = handler
self.server_event(events.Start())
await asyncio.wait([handler])
watch.cancel()
self.log("client disconnect")
self.client.timestamp_end = time.time()
await self.handle_hook(server_hooks.ClientDisconnectedHook(self.client)
)
if self.transports:
self.log("closing transports...", "debug")
for io in self.transports.values():
if io.handler:
asyncio_utils.cancel_task(io.handler,
"client disconnected")
await asyncio.wait(
[x.handler for x in self.transports.values() if x.handler])
self.log("transports closed!", "debug")
def make_stream(self, stream_id: int) -> layer.CommandGenerator[None]:
ctx = self.context.fork()
self.streams[stream_id] = HttpStream(ctx, stream_id)
yield from self.event_to_child(self.streams[stream_id], events.Start())
def get_connection(self,
event: GetHttpConnection,
*,
reuse: bool = True) -> layer.CommandGenerator[None]:
# Do we already have a connection we can re-use?
if reuse:
for connection in self.connections:
connection_suitable = (
event.connection_spec_matches(connection))
if connection_suitable:
if connection in self.waiting_for_establishment:
self.waiting_for_establishment[connection].append(
event)
return
elif connection.error:
stream = self.command_sources.pop(event)
yield from self.event_to_child(
stream,
GetHttpConnectionCompleted(
event, (None, connection.error)))
return
elif connection.connected:
# see "tricky multiplexing edge case" in make_http_connection for an explanation
h2_to_h1 = self.context.client.alpn == b"h2" and connection.alpn != b"h2"
if not h2_to_h1:
stream = self.command_sources.pop(event)
yield from self.event_to_child(
stream,
GetHttpConnectionCompleted(
event, (connection, None)))
return
else:
pass # the connection is at least half-closed already, we want a new one.
context_connection_matches = (
self.context.server not in self.connections
and event.connection_spec_matches(self.context.server))
can_use_context_connection = (context_connection_matches
and self.context.server.connected)
if context_connection_matches and self.context.server.error:
stream = self.command_sources.pop(event)
yield from self.event_to_child(
stream,
GetHttpConnectionCompleted(event,
(None, self.context.server.error)))
return
context = self.context.fork()
stack = tunnel.LayerStack()
if not can_use_context_connection:
context.server = Server(event.address)
if event.via:
context.server.via = event.via
assert event.via.scheme in ("http", "https")
# We always send a CONNECT request, *except* for plaintext absolute-form HTTP requests in upstream mode.
send_connect = event.tls or self.mode != HTTPMode.upstream
stack /= _upstream_proxy.HttpUpstreamProxy.make(
context, send_connect)
if event.tls:
# Assume that we are in transparent mode and lazily did not open a connection yet.
# We don't want the IP (which is the address) as the upstream SNI, but the client's SNI instead.
if self.mode == HTTPMode.transparent and event.address == self.context.server.address:
context.server.sni = self.context.client.sni or event.address[
0]
else:
context.server.sni = event.address[0]
stack /= tls.ServerTLSLayer(context)
stack /= HttpClient(context)
self.connections[context.server] = stack[0]
self.waiting_for_establishment[context.server].append(event)
yield from self.event_to_child(stack[0], events.Start())
def test_dataclasses(tconn):
assert repr(events.Start())
assert repr(events.DataReceived(tconn, b"foo"))
assert repr(events.ConnectionClosed(tconn))
async def replay(self) -> None:
self.server_event(events.Start())
await self.done.wait()