def socket_handler(listener):
sock = listener.accept()[0]
# Do the handshake: conn header, settings, send settings, recv ack.
receive_preamble(sock)
# Now expect some data. One headers frame and one data frame.
data.append(sock.recv(65535))
data.append(sock.recv(65535))
# Respond!
h = HeadersFrame(1)
h.data = self.get_encoder().encode([
(':status', 200),
('content-type', 'not/real'),
('content-length', 20),
])
h.flags.add('END_HEADERS')
sock.send(h.serialize())
d = DataFrame(1)
d.data = b'1234567890' * 2
d.flags.add('END_STREAM')
sock.send(d.serialize())
sock.close()
def socket_handler(listener):
sock = listener.accept()[0]
receive_preamble(sock)
data.append(sock.recv(65535))
send_event.wait(5)
h = HeadersFrame(1)
h.data = self.get_encoder().encode(
[
(':status', 200),
('content-type', 'not/real'),
('content-length', 12),
('server', 'socket-level-server')
]
)
h.flags.add('END_HEADERS')
sock.send(h.serialize())
d = DataFrame(1)
d.data = b'thisisaproxy'
d.flags.add('END_STREAM')
sock.send(d.serialize())
sock.close()
def socket_handler(listener):
sock = listener.accept()[0]
# Do the handshake: conn header, settings, send settings, recv ack.
receive_preamble(sock)
# Now expect some data. One headers frame.
data.append(sock.recv(65535))
# Respond!
h = HeadersFrame(1)
h.data = self.get_encoder().encode(
[
(':status', 200),
('content-type', 'not/real'),
('content-length', 20),
]
)
h.flags.add('END_HEADERS')
sock.send(h.serialize())
d = DataFrame(1)
d.data = b'1234567890' * 2
d.flags.add('END_STREAM')
sock.send(d.serialize())
send_event.wait(5)
sock.close()
def test_headers_frame_serializes_properly(self):
f = HeadersFrame(1)
f.flags = set(['END_STREAM', 'END_HEADERS'])
f.data = b'hello world'
s = f.serialize()
assert s == (b'\x00\x00\x0B\x01\x05\x00\x00\x00\x01' + b'hello world')
def test_stream_window_increments_appropriately(self, frame_buffer):
e = Encoder()
h = HeadersFrame(1)
h.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h.flags = set(['END_HEADERS'])
d = DataFrame(1)
d.data = b'hi there sir'
d2 = DataFrame(1)
d2.data = b'hi there sir again'
sock = DummySocket()
sock.buffer = BytesIO(h.serialize() + d.serialize() + d2.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
c.request('GET', '/')
c.streams[1]._in_window_manager.window_size = 1000
c.streams[1]._in_window_manager.initial_window_size = 1000
resp = c.get_response()
resp.read(len(b'hi there sir'))
resp.read(len(b'hi there sir again'))
frame_buffer.add_data(b''.join(sock.queue))
queue = list(frame_buffer)
assert len(queue) == 3 # one headers frame, two window update frames.
assert isinstance(queue[1], WindowUpdateFrame)
assert queue[1].window_increment == len(b'hi there sir')
assert isinstance(queue[2], WindowUpdateFrame)
assert queue[2].window_increment == len(b'hi there sir again')
def test_stream_window_increments_appropriately(self, frame_buffer):
e = Encoder()
h = HeadersFrame(1)
h.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h.flags = set(['END_HEADERS'])
d = DataFrame(1)
d.data = b'hi there sir'
d2 = DataFrame(1)
d2.data = b'hi there sir again'
sock = DummySocket()
sock.buffer = BytesIO(h.serialize() + d.serialize() + d2.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
c.request('GET', '/')
c.streams[1]._in_window_manager.window_size = 1000
c.streams[1]._in_window_manager.initial_window_size = 1000
resp = c.get_response()
resp.read(len(b'hi there sir'))
resp.read(len(b'hi there sir again'))
frame_buffer.add_data(b''.join(sock.queue))
queue = list(frame_buffer)
assert len(queue) == 3 # one headers frame, two window update frames.
assert isinstance(queue[1], WindowUpdateFrame)
assert queue[1].window_increment == len(b'hi there sir')
assert isinstance(queue[2], WindowUpdateFrame)
assert queue[2].window_increment == len(b'hi there sir again')
def test_headers_frame_with_no_length_parses(self):
# Fixes issue with empty data frames raising InvalidPaddingError.
f = HeadersFrame(1)
f.data = b''
data = f.serialize()
new_frame = decode_frame(data)
assert new_frame.data == b''
def test_headers_frame_with_no_length_parses(self):
# Fixes issue with empty data frames raising InvalidPaddingError.
f = HeadersFrame(1)
f.data = b''
data = f.serialize()
new_frame = decode_frame(data)
assert new_frame.data == b''
def build_headers_frame(headers, encoder=None):
f = HeadersFrame(1)
e = encoder
if e is None:
e = Encoder()
e.huffman_coder = HuffmanEncoder(REQUEST_CODES, REQUEST_CODES_LENGTH)
f.data = e.encode(headers)
f.flags.add('END_HEADERS')
return f
def build_headers_frame(headers, encoder=None):
f = HeadersFrame(1)
e = encoder
if e is None:
e = Encoder()
e.huffman_coder = HuffmanEncoder(REQUEST_CODES, REQUEST_CODES_LENGTH)
f.data = e.encode(headers)
f.flags.add('END_HEADERS')
return f
def add_headers_frame(self, stream_id, headers, end_block=True,
end_stream=False):
frame = HeadersFrame(stream_id)
frame.data = self.encoder.encode(headers)
if end_block:
frame.flags.add('END_HEADERS')
if end_stream:
frame.flags.add('END_STREAM')
self.frames.append(frame)
def add_headers_frame(self, stream_id, headers, end_block=True,
end_stream=False):
frame = HeadersFrame(stream_id)
frame.data = self.encoder.encode(headers)
if end_block:
frame.flags.add('END_HEADERS')
if end_stream:
frame.flags.add('END_STREAM')
self.frames.append(frame)
def build_headers_frame(self, headers, flags=[], stream_id=1):
"""
Builds a single valid headers frame out of the contained headers.
"""
f = HeadersFrame(stream_id)
f.data = self.encoder.encode(headers)
f.flags.add('END_HEADERS')
for flag in flags:
f.flags.add(flag)
return f
def build_headers_frame(self, headers, flags=[], stream_id=1):
"""
Builds a single valid headers frame out of the contained headers.
"""
f = HeadersFrame(stream_id)
f.data = self.encoder.encode(headers)
f.flags.add('END_HEADERS')
for flag in flags:
f.flags.add(flag)
return f
def test_headers_frame_serializes_properly(self):
f = HeadersFrame(1)
f.flags = set(['END_STREAM', 'END_HEADERS'])
f.data = b'hello world'
s = f.serialize()
assert s == (
b'\x00\x00\x0B\x01\x05\x00\x00\x00\x01' +
b'hello world'
)
def start_request(self):
"""
Open the stream. Does this by encoding and sending the headers: no more
calls to ``add_header`` are allowed after this method is called.
The `end` flag controls whether this will be the end of the stream, or
whether data will follow.
"""
# Strip any headers invalid in H2.
#headers = h2_safe_headers(self.request_headers)
host = self.connection.get_host(self.task.host)
self.add_header(":method", self.task.method)
self.add_header(":scheme", "https")
self.add_header(":authority", host)
self.add_header(":path", self.task.path)
default_headers = (':method', ':scheme', ':authority', ':path')
#headers = h2_safe_headers(self.task.headers)
for name, value in list(self.task.headers.items()):
is_default = to_native_string(name) in default_headers
self.add_header(name, value, replace=is_default)
# Encode the headers.
encoded_headers = self._encoder(self.request_headers)
# It's possible that there is a substantial amount of data here. The
# data needs to go into one HEADERS frame, followed by a number of
# CONTINUATION frames. For now, for ease of implementation, let's just
# assume that's never going to happen (16kB of headers is lots!).
# Additionally, since this is so unlikely, there's no point writing a
# test for this: it's just so simple.
if len(encoded_headers) > FRAME_MAX_LEN: # pragma: no cover
raise ValueError("Header block too large.")
header_frame = HeadersFrame(self.stream_id)
header_frame.data = encoded_headers
# If no data has been provided, this is the end of the stream. Either
# way, due to the restriction above it's definitely the end of the
# headers.
header_frame.flags.add('END_HEADERS')
if self.request_body_left == 0:
header_frame.flags.add('END_STREAM')
# Send the header frame.
self.task.set_state("start send header")
self._send_cb(header_frame)
# Transition the stream state appropriately.
self.state = STATE_OPEN
self.task.set_state("start send left body")
if self.request_body_left > 0:
self.send_left_body()
def test_incrementing_window_after_close(self):
"""
Hyper does not attempt to increment the flow control window once the
stream is closed.
"""
# For this test, we want to send a response that has three frames at
# the default max frame size (16,384 bytes). That will, on the third
# frame, trigger the processing to increment the flow control window,
# which should then not happen.
f = SettingsFrame(0, settings={h2.settings.INITIAL_WINDOW_SIZE: 100})
c = HTTP20Connection('www.google.com')
c._sock = DummySocket()
c._sock.buffer = BytesIO(f.serialize())
# Open stream 1.
c.request('GET', '/')
# Check what data we've sent right now.
originally_sent_data = c._sock.queue[:]
# Swap out the buffer to get a GoAway frame.
length = 16384
total_length = (3 * 16384) + len(b'some more data')
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode(
[(':status', 200), ('content-length', '%d' % total_length)]
)
h1.flags |= set(['END_HEADERS'])
d1 = DataFrame(1)
d1.data = b'\x00' * length
d2 = d1
d3 = d1
d4 = DataFrame(1)
d4.data = b'some more data'
d4.flags |= set(['END_STREAM'])
buffer = BytesIO(
b''.join(f.serialize() for f in [h1, d1, d2, d3, d4])
)
c._sock.buffer = buffer
# Read the response
resp = c.get_response(stream_id=1)
assert resp.status == 200
assert resp.read() == b''.join(
[b'\x00' * (3 * length), b'some more data']
)
# We should have sent only one extra frame
assert len(originally_sent_data) + 1 == len(c._sock.queue)
def test_incrementing_window_after_close(self):
"""
Hyper does not attempt to increment the flow control window once the
stream is closed.
"""
# For this test, we want to send a response that has three frames at
# the default max frame size (16,384 bytes). That will, on the third
# frame, trigger the processing to increment the flow control window,
# which should then not happen.
f = SettingsFrame(0, settings={h2.settings.INITIAL_WINDOW_SIZE: 100})
c = HTTP20Connection('www.google.com')
c._sock = DummySocket()
c._sock.buffer = BytesIO(f.serialize())
# Open stream 1.
c.request('GET', '/')
# Check what data we've sent right now.
originally_sent_data = c._sock.queue[:]
# Swap out the buffer to get a GoAway frame.
length = 16384
total_length = (3 * 16384) + len(b'some more data')
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode(
[(':status', 200), ('content-length', '%d' % total_length)]
)
h1.flags |= set(['END_HEADERS'])
d1 = DataFrame(1)
d1.data = b'\x00' * length
d2 = d1
d3 = d1
d4 = DataFrame(1)
d4.data = b'some more data'
d4.flags |= set(['END_STREAM'])
buffer = BytesIO(
b''.join(f.serialize() for f in [h1, d1, d2, d3, d4])
)
c._sock.buffer = buffer
# Read the response
resp = c.get_response(stream_id=1)
assert resp.status == 200
assert resp.read() == b''.join(
[b'\x00' * (3 * length), b'some more data']
)
# We should have sent only one extra frame
assert len(originally_sent_data) + 1 == len(c._sock.queue)
def test_read_headers_out_of_order(self):
# If header blocks aren't decoded in the same order they're received,
# regardless of the stream they belong to, the decoder state will
# become corrupted.
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h1.flags |= set(['END_HEADERS', 'END_STREAM'])
h3 = HeadersFrame(3)
h3.data = e.encode([(':status', 200), ('content-type', 'baz/qux')])
h3.flags |= set(['END_HEADERS', 'END_STREAM'])
sock = DummySocket()
sock.buffer = BytesIO(h1.serialize() + h3.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
r1 = c.request('GET', '/a')
r3 = c.request('GET', '/b')
assert c.get_response(r3).headers == HTTPHeaderMap(
[('content-type', 'baz/qux')]
)
assert c.get_response(r1).headers == HTTPHeaderMap(
[('content-type', 'foo/bar')]
)
def send_headers(self, headers, encoder, end_stream=False):
"""
Returns a list of HEADERS/CONTINUATION frames to emit as either headers
or trailers.
"""
# Because encoding headers makes an irreversible change to the header
# compression context, we make the state transition *first*.
# This does not trigger any events.
events = self.state_machine.process_input(StreamInputs.SEND_HEADERS)
assert not events
hf = HeadersFrame(self.stream_id)
frames = self._build_headers_frames(headers, encoder, hf)
if end_stream:
# Not a bug: the END_STREAM flag is valid on the initial HEADERS
# frame, not the CONTINUATION frames that follow.
self.state_machine.process_input(StreamInputs.SEND_END_STREAM)
frames[0].flags.add('END_STREAM')
if self.state_machine.trailers_sent and not end_stream:
raise ProtocolError("Trailers must have END_STREAM set.")
return frames
def slow_headers(self, conn, h2conn, method="GET"):
LOGGER.info("SLOW HEADERS ATTACK=============================")
h2conn.initiate_connection()
wf = WindowUpdateFrame(0)
wf.window_increment = WINDOW_SIZE_INCREMENT
h2conn._data_to_send += wf.serialize()
conn.sendall(h2conn.data_to_send())
headers = [(":authority", args.target), (":path", "/"),
(":scheme", "http"), (":method", method)]
hf = HeadersFrame(1)
if method == "GET":
hf.flags.add("END_STREAM")
e = Encoder()
hf.data = e.encode(headers)
h2conn._data_to_send += hf.serialize()
conn.sendall(h2conn.data_to_send())
def send_headers(self, headers, encoder, end_stream=False):
"""
Returns a list of HEADERS/CONTINUATION frames to emit as either headers
or trailers.
"""
# Convert headers to two-tuples.
# FIXME: The fallback for dictionary headers is to be removed in 3.0.
try:
headers = headers.items()
warnings.warn(
"Implicit conversion of dictionaries to two-tuples for "
"headers is deprecated and will be removed in 3.0.",
DeprecationWarning
)
except AttributeError:
headers = headers
# Because encoding headers makes an irreversible change to the header
# compression context, we make the state transition before we encode
# them.
# First, check if we're a client. If we are, no problem: if we aren't,
# we need to scan the header block to see if this is an informational
# response.
input_ = StreamInputs.SEND_HEADERS
if ((not self.state_machine.client) and
is_informational_response(headers)):
if end_stream:
raise ProtocolError(
"Cannot set END_STREAM on informational responses."
)
input_ = StreamInputs.SEND_INFORMATIONAL_HEADERS
events = self.state_machine.process_input(input_)
hf = HeadersFrame(self.stream_id)
hdr_validation_flags = self._build_hdr_validation_flags(events)
frames = self._build_headers_frames(
headers, encoder, hf, hdr_validation_flags
)
if end_stream:
# Not a bug: the END_STREAM flag is valid on the initial HEADERS
# frame, not the CONTINUATION frames that follow.
self.state_machine.process_input(StreamInputs.SEND_END_STREAM)
frames[0].flags.add('END_STREAM')
if self.state_machine.trailers_sent and not end_stream:
raise ProtocolError("Trailers must have END_STREAM set.")
if self.state_machine.client and self._authority is None:
self._authority = authority_from_headers(headers)
# store request method for _initialize_content_length
self.request_method = extract_method_header(headers)
return frames
def attack2(tls_conn, h2_conn):
h2_conn.initiate_connection()
wf = WindowUpdateFrame(0)
wf.window_increment = WINDOW_INCREMENT_SIZE
h2_conn._data_to_send += wf.serialize()
tls_conn.sendall(h2_conn.data_to_send())
headers = [
(':authority', args.target),
(':path', '/'),
(':scheme', 'https'),
(':method', 'POST'),
]
hf = HeadersFrame(1)
hf.flags.add('END_HEADERS')
e = Encoder()
hf.data = hf.data = e.encode(headers)
h2_conn._data_to_send += hf.serialize()
tls_conn.sendall(h2_conn.data_to_send())
def build_headers_frame(self,
headers,
flags=[],
stream_id=1,
**priority_kwargs):
"""
Builds a single valid headers frame out of the contained headers.
"""
f = HeadersFrame(stream_id)
f.data = self.encoder.encode(headers)
f.flags.add('END_HEADERS')
for flag in flags:
f.flags.add(flag)
for k, v in priority_kwargs.items():
setattr(f, k, v)
return f
def build_headers_frame(self,
headers,
flags=[],
stream_id=1,
**priority_kwargs):
"""
Builds a single valid headers frame out of the contained headers.
"""
f = HeadersFrame(stream_id)
f.data = self.encoder.encode(headers)
f.flags.add('END_HEADERS')
for flag in flags:
f.flags.add(flag)
for k, v in priority_kwargs.items():
setattr(f, k, v)
return f
def create_socket(status_code, data, headers):
# test helper method
encoder = Encoder()
h1 = HeadersFrame(1)
h1.data = encoder.encode([(':status', status_code),
('content-length', len(data))] + headers)
h1.flags |= set(['END_HEADERS'])
d1 = DataFrame(1)
d1.data = data
d2 = DataFrame(1)
d2.flags |= set(['END_STREAM'])
content = b''.join(f.serialize() for f in [h1, d1, d2])
buffer = BytesIO(content)
return DummySocket(buffer)
def test_headers_frame_with_priority_serializes_properly(self):
# This test also tests that we can receive a HEADERS frame with no
# actual headers on it. This is technically possible.
s = (b'\x00\x00\x05\x01\x20\x00\x00\x00\x01' + b'\x80\x00\x00\x04\x40')
f = HeadersFrame(1)
f.flags = set(['PRIORITY'])
f.data = b''
f.depends_on = 4
f.stream_weight = 64
f.exclusive = True
assert f.serialize() == s
def test_headers_with_continuation(self):
e = Encoder()
header_data = e.encode([(':status', 200), ('content-type', 'foo/bar'),
('content-length', '0')])
h = HeadersFrame(1)
h.data = header_data[0:int(len(header_data) / 2)]
h.flags.add('END_STREAM')
c = ContinuationFrame(1)
c.data = header_data[int(len(header_data) / 2):]
c.flags.add('END_HEADERS')
sock = DummySocket()
sock.buffer = BytesIO(h.serialize() + c.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
r = c.request('GET', '/')
assert set(c.get_response(r).headers.iter_raw()) == set([
(b'content-type', b'foo/bar'), (b'content-length', b'0')
])
def test_read_headers_out_of_order(self):
# If header blocks aren't decoded in the same order they're received,
# regardless of the stream they belong to, the decoder state will
# become corrupted.
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h1.flags |= set(['END_HEADERS', 'END_STREAM'])
h3 = HeadersFrame(3)
h3.data = e.encode([(':status', 200), ('content-type', 'baz/qux')])
h3.flags |= set(['END_HEADERS', 'END_STREAM'])
sock = DummySocket()
sock.buffer = BytesIO(h1.serialize() + h3.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
r1 = c.request('GET', '/a')
r3 = c.request('GET', '/b')
assert c.get_response(r3).headers == HTTPHeaderMap(
[('content-type', 'baz/qux')]
)
assert c.get_response(r1).headers == HTTPHeaderMap(
[('content-type', 'foo/bar')]
)
def test_headers_with_continuation(self):
e = Encoder()
header_data = e.encode([
(':status', 200), ('content-type', 'foo/bar'),
('content-length', '0')
])
h = HeadersFrame(1)
h.data = header_data[0:int(len(header_data)/2)]
h.flags.add('END_STREAM')
c = ContinuationFrame(1)
c.data = header_data[int(len(header_data)/2):]
c.flags.add('END_HEADERS')
sock = DummySocket()
sock.buffer = BytesIO(h.serialize() + c.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
r = c.request('GET', '/')
assert set(c.get_response(r).headers.iter_raw()) == set(
[(b'content-type', b'foo/bar'), (b'content-length', b'0')]
)
def test_streams_removed_on_close(self):
# Create content for read from socket
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h1.flags |= set(['END_HEADERS', 'END_STREAM'])
sock = DummySocket()
sock.buffer = BytesIO(h1.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
stream_id = c.request('GET', '/')
# Create reference to current recent_recv_streams set
recent_recv_streams = c.recent_recv_streams
streams = c.streams
resp = c.get_response(stream_id=stream_id)
assert stream_id in recent_recv_streams
assert stream_id in streams
resp.read()
assert stream_id not in recent_recv_streams
assert stream_id not in streams
def socket_handler(listener):
sock = listener.accept()[0]
receive_preamble(sock)
data.append(sock.recv(65535))
send_event.wait(5)
h = HeadersFrame(1)
h.data = self.get_encoder().encode([(':status', 200),
('content-type', 'not/real'),
('content-length', 12),
('server',
'socket-level-server')])
h.flags.add('END_HEADERS')
sock.send(h.serialize())
d = DataFrame(1)
d.data = b'thisisaproxy'
d.flags.add('END_STREAM')
sock.send(d.serialize())
sock.close()
def test_streams_removed_on_close(self):
# Create content for read from socket
e = Encoder()
h1 = HeadersFrame(1)
h1.data = e.encode([(':status', 200), ('content-type', 'foo/bar')])
h1.flags |= set(['END_HEADERS', 'END_STREAM'])
sock = DummySocket()
sock.buffer = BytesIO(h1.serialize())
c = HTTP20Connection('www.google.com')
c._sock = sock
stream_id = c.request('GET', '/')
# Create reference to current recent_recv_streams set
recent_recv_streams = c.recent_recv_streams
streams = c.streams
resp = c.get_response(stream_id=stream_id)
assert stream_id in recent_recv_streams
assert stream_id in streams
resp.read()
assert stream_id not in recent_recv_streams
assert stream_id not in streams
def test_repr(self):
f = HeadersFrame(1)
assert repr(f).endswith(
"exclusive=False, depends_on=0, stream_weight=0, data=None")
f.data = b'hello'
f.exclusive = True
f.depends_on = 42
f.stream_weight = 64
assert repr(f).endswith(
"exclusive=True, depends_on=42, stream_weight=64, data=<hex:68656c6c6f>"
)
def test_headers_frame_with_priority_serializes_properly(self):
# This test also tests that we can receive a HEADERS frame with no
# actual headers on it. This is technically possible.
s = (
b'\x00\x00\x05\x01\x20\x00\x00\x00\x01' +
b'\x80\x00\x00\x04\x40'
)
f = HeadersFrame(1)
f.flags = set(['PRIORITY'])
f.data = b''
f.depends_on = 4
f.stream_weight = 64
f.exclusive = True
assert f.serialize() == s
def test_headers_frame_flags(self):
f = HeadersFrame(1)
flags = f.parse_flags(0xFF)
assert flags == set(['END_STREAM', 'END_HEADERS',
'PADDED', 'PRIORITY'])
def test_headers_frame_flags(self):
f = HeadersFrame(1)
flags = f.parse_flags(0xFF)
assert flags == set(
['END_STREAM', 'END_HEADERS', 'PADDED', 'PRIORITY'])
