def handle_event(self, event: QuicEvent) -> List[H3Event]: http_events: List[H3Event] = [] if isinstance(event, StreamDataReceived) and (event.stream_id % 4) == 0: data = self._buffer.pop(event.stream_id, b"") + event.data if not self._headers_received.get(event.stream_id, False): if self._is_client: http_events.append( HeadersReceived(headers=[], stream_ended=False, stream_id=event.stream_id)) elif data.endswith(b"\r\n") or event.end_stream: method, path = data.rstrip().split(b" ", 1) http_events.append( HeadersReceived( headers=[(b":method", method), (b":path", path)], stream_ended=False, stream_id=event.stream_id, )) data = b"" else: # incomplete request, stash the data self._buffer[event.stream_id] = data return http_events self._headers_received[event.stream_id] = True http_events.append( DataReceived(data=data, stream_ended=event.end_stream, stream_id=event.stream_id)) return http_events
def handle_event(self, event: QuicEvent) -> List[H3Event]: http_events: List[H3Event] = [] if isinstance(event, StreamDataReceived) and (event.stream_id % 4) == 0: data = event.data if not self._headers_received.get(event.stream_id, False): if self._is_client: http_events.append( HeadersReceived( headers=[], stream_ended=False, stream_id=event.stream_id ) ) else: method, path = data.rstrip().split(b" ", 1) http_events.append( HeadersReceived( headers=[(b":method", method), (b":path", path)], stream_ended=False, stream_id=event.stream_id, ) ) data = b"" self._headers_received[event.stream_id] = True http_events.append( DataReceived( data=data, stream_ended=event.end_stream, stream_id=event.stream_id ) ) return http_events
def test_blocked_stream(self): quic_client = FakeQuicConnection(configuration=QuicConfiguration( is_client=True)) h3_client = H3Connection(quic_client) h3_client.handle_event( StreamDataReceived( stream_id=3, data=binascii.unhexlify( "0004170150000680020000074064091040bcc0000000faceb00c"), end_stream=False, )) h3_client.handle_event( StreamDataReceived(stream_id=7, data=b"\x02", end_stream=False)) h3_client.handle_event( StreamDataReceived(stream_id=11, data=b"\x03", end_stream=False)) h3_client.handle_event( StreamDataReceived(stream_id=0, data=binascii.unhexlify("01040280d910"), end_stream=False)) h3_client.handle_event( StreamDataReceived( stream_id=0, data=binascii.unhexlify( "00408d796f752072656163686564206d766673742e6e65742c20726561636820" "746865202f6563686f20656e64706f696e7420666f7220616e206563686f2072" "6573706f6e7365207175657279202f3c6e756d6265723e20656e64706f696e74" "7320666f722061207661726961626c652073697a6520726573706f6e73652077" "6974682072616e646f6d206279746573"), end_stream=True, )) self.assertEqual( h3_client.handle_event( StreamDataReceived( stream_id=7, data=binascii.unhexlify( "3fe101c696d07abe941094cb6d0a08017d403971966e32ca98b46f" ), end_stream=False, )), [ HeadersReceived( headers=[ (b":status", b"200"), (b"date", b"Mon, 22 Jul 2019 06:33:33 GMT"), ], stream_id=0, stream_ended=False, ), DataReceived( data= (b"you reached mvfst.net, reach the /echo endpoint for an " b"echo response query /<number> endpoints for a variable " b"size response with random bytes"), stream_id=0, stream_ended=True, ), ], )
def _receive_request_or_push_data(self, stream: H3Stream, data: bytes, stream_ended: bool) -> List[H3Event]: """ Handle data received on a request or push stream. """ http_events: List[H3Event] = [] stream.buffer += data if stream_ended: stream.ended = True if stream.blocked: return http_events # shortcut for DATA frame fragments if (stream.frame_type == FrameType.DATA and stream.frame_size is not None and len(stream.buffer) < stream.frame_size): http_events.append( DataReceived( data=stream.buffer, push_id=stream.push_id, stream_id=stream.stream_id, stream_ended=False, )) stream.frame_size -= len(stream.buffer) stream.buffer = b"" return http_events # handle lone FIN if stream_ended and not stream.buffer: http_events.append( DataReceived( data=b"", push_id=stream.push_id, stream_id=stream.stream_id, stream_ended=True, )) return http_events buf = Buffer(data=stream.buffer) consumed = 0 while not buf.eof(): # fetch next frame header if stream.frame_size is None: try: stream.frame_type = buf.pull_uint_var() stream.frame_size = buf.pull_uint_var() except BufferReadError: break consumed = buf.tell() # log frame if (self._quic_logger is not None and stream.frame_type == FrameType.DATA): self._quic_logger.log_event( category="http", event="frame_parsed", data=qlog_encode_data_frame( byte_length=stream.frame_size, stream_id=stream.stream_id), ) # check how much data is available chunk_size = min(stream.frame_size, buf.capacity - consumed) if stream.frame_type != FrameType.DATA and chunk_size < stream.frame_size: break # read available data frame_data = buf.pull_bytes(chunk_size) consumed = buf.tell() # detect end of frame stream.frame_size -= chunk_size if not stream.frame_size: stream.frame_size = None try: http_events.extend( self._handle_request_or_push_frame( frame_type=stream.frame_type, frame_data=frame_data, stream=stream, stream_ended=stream.ended and buf.eof(), )) except pylsqpack.StreamBlocked: stream.blocked = True stream.blocked_frame_size = len(frame_data) break # remove processed data from buffer stream.buffer = stream.buffer[consumed:] return http_events
def _handle_request_or_push_frame( self, frame_type: int, frame_data: Optional[bytes], stream: H3Stream, stream_ended: bool, ) -> List[H3Event]: """ Handle a frame received on a request or push stream. """ http_events: List[H3Event] = [] if frame_type == FrameType.DATA: # check DATA frame is allowed if stream.headers_recv_state != HeadersState.AFTER_HEADERS: raise FrameUnexpected( "DATA frame is not allowed in this state") if stream_ended or frame_data: http_events.append( DataReceived( data=frame_data, push_id=stream.push_id, stream_ended=stream_ended, stream_id=stream.stream_id, )) elif frame_type == FrameType.HEADERS: # check HEADERS frame is allowed if stream.headers_recv_state == HeadersState.AFTER_TRAILERS: raise FrameUnexpected( "HEADERS frame is not allowed in this state") # try to decode HEADERS, may raise pylsqpack.StreamBlocked headers = self._decode_headers(stream.stream_id, frame_data) # log frame if self._quic_logger is not None: self._quic_logger.log_event( category="http", event="frame_parsed", data=qlog_encode_headers_frame( byte_length=stream.blocked_frame_size if frame_data is None else len(frame_data), headers=headers, stream_id=stream.stream_id, ), ) # update state and emit headers if stream.headers_recv_state == HeadersState.INITIAL: stream.headers_recv_state = HeadersState.AFTER_HEADERS else: stream.headers_recv_state = HeadersState.AFTER_TRAILERS http_events.append( HeadersReceived( headers=headers, push_id=stream.push_id, stream_id=stream.stream_id, stream_ended=stream_ended, )) elif stream.frame_type == FrameType.PUSH_PROMISE and stream.push_id is None: if not self._is_client: raise FrameUnexpected("Clients must not send PUSH_PROMISE") frame_buf = Buffer(data=frame_data) push_id = frame_buf.pull_uint_var() headers = self._decode_headers(stream.stream_id, frame_data[frame_buf.tell():]) # log frame if self._quic_logger is not None: self._quic_logger.log_event( category="http", event="frame_parsed", data=qlog_encode_push_promise_frame( byte_length=len(frame_data), headers=headers, push_id=push_id, stream_id=stream.stream_id, ), ) # emit event http_events.append( PushPromiseReceived(headers=headers, push_id=push_id, stream_id=stream.stream_id)) elif frame_type in ( FrameType.PRIORITY, FrameType.CANCEL_PUSH, FrameType.SETTINGS, FrameType.PUSH_PROMISE, FrameType.GOAWAY, FrameType.MAX_PUSH_ID, FrameType.DUPLICATE_PUSH, ): raise FrameUnexpected( "Invalid frame type on request stream" if stream. push_id is None else "Invalid frame type on push stream") return http_events
def test_request_with_server_push(self): with client_and_server( client_options={"alpn_protocols": H3_ALPN}, server_options={"alpn_protocols": H3_ALPN}, ) as (quic_client, quic_server): h3_client = H3Connection(quic_client) h3_server = H3Connection(quic_server) # send request stream_id = quic_client.get_next_available_stream_id() h3_client.send_headers( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), ], end_stream=True, ) # receive request events = h3_transfer(quic_client, h3_server) self.assertEqual( events, [ HeadersReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), ], stream_id=stream_id, stream_ended=True, ) ], ) # send push promises push_stream_id_css = h3_server.send_push_promise( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/app.css"), ], ) self.assertEqual(push_stream_id_css, 15) push_stream_id_js = h3_server.send_push_promise( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/app.js"), ], ) self.assertEqual(push_stream_id_js, 19) # send response h3_server.send_headers( stream_id=stream_id, headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), ], end_stream=False, ) h3_server.send_data( stream_id=stream_id, data=b"<html><body>hello</body></html>", end_stream=True, ) # fulfill push promises h3_server.send_headers( stream_id=push_stream_id_css, headers=[(b":status", b"200"), (b"content-type", b"text/css")], end_stream=False, ) h3_server.send_data( stream_id=push_stream_id_css, data=b"body { color: pink }", end_stream=True, ) h3_server.send_headers( stream_id=push_stream_id_js, headers=[ (b":status", b"200"), (b"content-type", b"application/javascript"), ], end_stream=False, ) h3_server.send_data(stream_id=push_stream_id_js, data=b"alert('howdee');", end_stream=True) # receive push promises, response and push responses events = h3_transfer(quic_server, h3_client) self.assertEqual( events, [ PushPromiseReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/app.css"), ], push_id=0, stream_id=stream_id, ), PushPromiseReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/app.js"), ], push_id=1, stream_id=stream_id, ), HeadersReceived( headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), ], stream_id=stream_id, stream_ended=False, ), DataReceived( data=b"<html><body>hello</body></html>", stream_id=stream_id, stream_ended=True, ), HeadersReceived( headers=[(b":status", b"200"), (b"content-type", b"text/css")], push_id=0, stream_id=push_stream_id_css, stream_ended=False, ), DataReceived( data=b"body { color: pink }", push_id=0, stream_id=push_stream_id_css, stream_ended=True, ), HeadersReceived( headers=[ (b":status", b"200"), (b"content-type", b"application/javascript"), ], push_id=1, stream_id=push_stream_id_js, stream_ended=False, ), DataReceived( data=b"alert('howdee');", push_id=1, stream_id=push_stream_id_js, stream_ended=True, ), ], )
def _make_request(self, h3_client, h3_server): quic_client = h3_client._quic quic_server = h3_server._quic # send request stream_id = quic_client.get_next_available_stream_id() h3_client.send_headers( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), (b"x-foo", b"client"), ], ) h3_client.send_data(stream_id=stream_id, data=b"", end_stream=True) # receive request events = h3_transfer(quic_client, h3_server) self.assertEqual( events, [ HeadersReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), (b"x-foo", b"client"), ], stream_id=stream_id, stream_ended=False, ), DataReceived(data=b"", stream_id=stream_id, stream_ended=True), ], ) # send response h3_server.send_headers( stream_id=stream_id, headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), (b"x-foo", b"server"), ], ) h3_server.send_data( stream_id=stream_id, data=b"<html><body>hello</body></html>", end_stream=True, ) # receive response events = h3_transfer(quic_server, h3_client) self.assertEqual( events, [ HeadersReceived( headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), (b"x-foo", b"server"), ], stream_id=stream_id, stream_ended=False, ), DataReceived( data=b"<html><body>hello</body></html>", stream_id=stream_id, stream_ended=True, ), ], )
def test_request_fragmented_frame(self): quic_client = FakeQuicConnection(configuration=QuicConfiguration( is_client=True)) quic_server = FakeQuicConnection(configuration=QuicConfiguration( is_client=False)) h3_client = H3Connection(quic_client) h3_server = H3Connection(quic_server) # send request stream_id = quic_client.get_next_available_stream_id() h3_client.send_headers( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), (b"x-foo", b"client"), ], ) h3_client.send_data(stream_id=stream_id, data=b"hello", end_stream=True) # receive request events = h3_transfer(quic_client, h3_server) self.assertEqual( events, [ HeadersReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), (b"x-foo", b"client"), ], stream_id=stream_id, stream_ended=False, ), DataReceived(data=b"h", stream_id=0, stream_ended=False), DataReceived(data=b"e", stream_id=0, stream_ended=False), DataReceived(data=b"l", stream_id=0, stream_ended=False), DataReceived(data=b"l", stream_id=0, stream_ended=False), DataReceived(data=b"o", stream_id=0, stream_ended=False), DataReceived(data=b"", stream_id=0, stream_ended=True), ], ) # send push promise push_stream_id = h3_server.send_push_promise( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/app.txt"), ], ) self.assertEqual(push_stream_id, 15) # send response h3_server.send_headers( stream_id=stream_id, headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), ], end_stream=False, ) h3_server.send_data(stream_id=stream_id, data=b"html", end_stream=True) # fulfill push promise h3_server.send_headers( stream_id=push_stream_id, headers=[(b":status", b"200"), (b"content-type", b"text/plain")], end_stream=False, ) h3_server.send_data(stream_id=push_stream_id, data=b"text", end_stream=True) # receive push promise / reponse events = h3_transfer(quic_server, h3_client) self.assertEqual( events, [ PushPromiseReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/app.txt"), ], push_id=0, stream_id=stream_id, ), HeadersReceived( headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), ], stream_id=0, stream_ended=False, ), DataReceived(data=b"h", stream_id=0, stream_ended=False), DataReceived(data=b"t", stream_id=0, stream_ended=False), DataReceived(data=b"m", stream_id=0, stream_ended=False), DataReceived(data=b"l", stream_id=0, stream_ended=False), DataReceived(data=b"", stream_id=0, stream_ended=True), HeadersReceived( headers=[(b":status", b"200"), (b"content-type", b"text/plain")], stream_id=15, stream_ended=False, push_id=0, ), DataReceived( data=b"t", stream_id=15, stream_ended=False, push_id=0), DataReceived( data=b"e", stream_id=15, stream_ended=False, push_id=0), DataReceived( data=b"x", stream_id=15, stream_ended=False, push_id=0), DataReceived( data=b"t", stream_id=15, stream_ended=False, push_id=0), DataReceived( data=b"", stream_id=15, stream_ended=True, push_id=0), ], )
def test_request_with_trailers(self): with client_and_server( client_options={"alpn_protocols": H3_ALPN}, server_options={"alpn_protocols": H3_ALPN}, ) as (quic_client, quic_server): h3_client = H3Connection(quic_client) h3_server = H3Connection(quic_server) # send request with trailers stream_id = quic_client.get_next_available_stream_id() h3_client.send_headers( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), ], end_stream=False, ) h3_client.send_headers( stream_id=stream_id, headers=[(b"x-some-trailer", b"foo")], end_stream=True, ) # receive request events = h3_transfer(quic_client, h3_server) self.assertEqual( events, [ HeadersReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), ], stream_id=stream_id, stream_ended=False, ), HeadersReceived( headers=[(b"x-some-trailer", b"foo")], stream_id=stream_id, stream_ended=True, ), ], ) # send response h3_server.send_headers( stream_id=stream_id, headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), ], end_stream=False, ) h3_server.send_data( stream_id=stream_id, data=b"<html><body>hello</body></html>", end_stream=False, ) h3_server.send_headers( stream_id=stream_id, headers=[(b"x-some-trailer", b"bar")], end_stream=True, ) # receive response events = h3_transfer(quic_server, h3_client) self.assertEqual( events, [ HeadersReceived( headers=[ (b":status", b"200"), (b"content-type", b"text/html; charset=utf-8"), ], stream_id=stream_id, stream_ended=False, ), DataReceived( data=b"<html><body>hello</body></html>", stream_id=stream_id, stream_ended=False, ), HeadersReceived( headers=[(b"x-some-trailer", b"bar")], stream_id=stream_id, stream_ended=True, ), ], )
def _receive_stream_data( self, stream_id: int, data: bytes, stream_ended: bool ) -> List[Event]: http_events: List[Event] = [] if stream_id in self._stream_buffers: self._stream_buffers[stream_id] += data else: self._stream_buffers[stream_id] = data consumed = 0 buf = Buffer(data=self._stream_buffers[stream_id]) while not buf.eof(): # fetch stream type for unidirectional streams if ( stream_is_unidirectional(stream_id) and stream_id not in self._stream_types ): try: stream_type = buf.pull_uint_var() except BufferReadError: break consumed = buf.tell() if stream_type == StreamType.CONTROL: assert self._peer_control_stream_id is None self._peer_control_stream_id = stream_id elif stream_type == StreamType.QPACK_DECODER: assert self._peer_decoder_stream_id is None self._peer_decoder_stream_id = stream_id elif stream_type == StreamType.QPACK_ENCODER: assert self._peer_encoder_stream_id is None self._peer_encoder_stream_id = stream_id self._stream_types[stream_id] = stream_type # fetch next frame try: frame_type = buf.pull_uint_var() frame_length = buf.pull_uint_var() frame_data = buf.pull_bytes(frame_length) except BufferReadError: break consumed = buf.tell() if (stream_id % 4) == 0: # client-initiated bidirectional streams carry requests and responses if frame_type == FrameType.DATA: http_events.append( DataReceived( data=frame_data, stream_id=stream_id, stream_ended=stream_ended and buf.eof(), ) ) elif frame_type == FrameType.HEADERS: control, headers = self._decoder.feed_header(stream_id, frame_data) cls = ResponseReceived if self._is_client else RequestReceived http_events.append( cls( headers=headers, stream_id=stream_id, stream_ended=stream_ended and buf.eof(), ) ) elif stream_id == self._peer_control_stream_id: # unidirectional control stream if frame_type == FrameType.SETTINGS: settings = parse_settings(frame_data) self._encoder.apply_settings( max_table_capacity=settings.get( Setting.QPACK_MAX_TABLE_CAPACITY, 0 ), blocked_streams=settings.get(Setting.QPACK_BLOCKED_STREAMS, 0), ) # remove processed data from buffer self._stream_buffers[stream_id] = self._stream_buffers[stream_id][consumed:] return http_events
def _receive_stream_data_bidi(self, stream_id: int, data: bytes, stream_ended: bool) -> List[HttpEvent]: """ Client-initiated bidirectional streams carry requests and responses. """ http_events: List[HttpEvent] = [] stream = self._stream[stream_id] stream.buffer += data if stream_ended: stream.ended = True if stream.blocked: return http_events # shortcut DATA frame bits if (stream.frame_size is not None and stream.frame_type == FrameType.DATA and len(stream.buffer) < stream.frame_size): http_events.append( DataReceived(data=stream.buffer, stream_id=stream_id, stream_ended=False)) stream.frame_size -= len(stream.buffer) stream.buffer = b"" return http_events # some peers (e.g. f5) end the stream with no data if stream_ended and not stream.buffer: http_events.append( DataReceived(data=b"", stream_id=stream_id, stream_ended=True)) return http_events buf = Buffer(data=stream.buffer) consumed = 0 while not buf.eof(): # fetch next frame header if stream.frame_size is None: try: stream.frame_type = buf.pull_uint_var() stream.frame_size = buf.pull_uint_var() except BufferReadError: break consumed = buf.tell() # check how much data is available chunk_size = min(stream.frame_size, buf.capacity - consumed) if (stream.frame_type == FrameType.HEADERS and chunk_size < stream.frame_size): break # read available data frame_data = buf.pull_bytes(chunk_size) consumed = buf.tell() # detect end of frame stream.frame_size -= chunk_size if not stream.frame_size: stream.frame_size = None if stream.frame_type == FrameType.DATA and (stream_ended or frame_data): http_events.append( DataReceived( data=frame_data, stream_id=stream_id, stream_ended=stream_ended and buf.eof(), )) elif stream.frame_type == FrameType.HEADERS: try: decoder, headers = self._decoder.feed_header( stream_id, frame_data) except StreamBlocked: stream.blocked = True break self._quic.send_stream_data(self._local_decoder_stream_id, decoder) cls = ResponseReceived if self._is_client else RequestReceived http_events.append( cls( headers=headers, stream_id=stream_id, stream_ended=stream_ended and buf.eof(), )) # remove processed data from buffer stream.buffer = stream.buffer[consumed:] return http_events
def test_fragmented_frame(self): quic_client = FakeQuicConnection(configuration=QuicConfiguration( is_client=True)) quic_server = FakeQuicConnection(configuration=QuicConfiguration( is_client=False)) h3_client = H3Connection(quic_client) h3_server = H3Connection(quic_server) # send headers stream_id = quic_client.get_next_available_stream_id() h3_client.send_headers( stream_id=stream_id, headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), (b"x-foo", b"client"), ], ) http_events = [] for event in quic_client.stream_queue: http_events.extend(h3_server.handle_event(event)) quic_client.stream_queue.clear() self.assertEqual( http_events, [ RequestReceived( headers=[ (b":method", b"GET"), (b":scheme", b"https"), (b":authority", b"localhost"), (b":path", b"/"), (b"x-foo", b"client"), ], stream_id=0, stream_ended=False, ) ], ) # send body h3_client.send_data(stream_id=stream_id, data=b"hello", end_stream=True) http_events = [] for event in quic_client.stream_queue: http_events.extend(h3_server.handle_event(event)) quic_client.stream_queue.clear() self.assertEqual( http_events, [ DataReceived(data=b"h", stream_id=0, stream_ended=False), DataReceived(data=b"e", stream_id=0, stream_ended=False), DataReceived(data=b"l", stream_id=0, stream_ended=False), DataReceived(data=b"l", stream_id=0, stream_ended=False), DataReceived(data=b"o", stream_id=0, stream_ended=False), DataReceived(data=b"", stream_id=0, stream_ended=True), ], )