def test_http2_connection_attempt_close():
"""
A connection can only be closed when it is idle.
"""
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=1, data=b"Hello, world!", flags=["END_STREAM"]
).serialize(),
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
with conn.stream("GET", "https://example.com/") as response:
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
conn.close()
with pytest.raises(ConnectionNotAvailable):
conn.request("GET", "https://example.com/")
def test_http2_connection_post_request():
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=1, data=b"Hello, world!", flags=["END_STREAM"]
).serialize(),
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
response = conn.request(
"POST",
"https://example.com/",
headers={b"content-length": b"17"},
content=b'{"data": "upload"}',
)
assert response.status == 200
assert response.content == b"Hello, world!"
async def test_http2_connection():
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
])
async with AsyncHTTP2Connection(origin=origin,
stream=stream,
keepalive_expiry=5.0) as conn:
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
assert conn.is_idle()
assert conn.is_available()
assert not conn.is_closed()
assert not conn.has_expired()
assert (conn.info() ==
"'https://example.com:443', HTTP/2, IDLE, Request Count: 1")
assert (
repr(conn) ==
"<AsyncHTTP2Connection ['https://example.com:443', IDLE, Request Count: 1]>"
)
async def test_http2_connection():
origin = Origin(b"https", b"example.com", 443)
network_backend = AsyncMockBackend(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
],
http2=True,
)
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend,
http2=True) as conn:
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
assert response.extensions["http_version"] == b"HTTP/2"
def test_http2_connection_with_flow_control():
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
# Available flow: 65,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 75,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 85,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 95,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 105,535
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=1, data=b"100,000 bytes received", flags=["END_STREAM"]
).serialize(),
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
response = conn.request(
"POST",
"https://example.com/",
content=b"x" * 100_000,
)
assert response.status == 200
assert response.content == b"100,000 bytes received"
def test_http2_connection_with_goaway():
"""
If a stream reset occurs, then no response will be returned,
but the connection will remain reusable for other requests.
"""
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
# Connection is closed midway through the first response...
hyperframe.frame.GoAwayFrame(stream_id=0, error_code=0).serialize(),
# ...We'll never get to this second response.
hyperframe.frame.HeadersFrame(
stream_id=3,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=3, data=b"Hello, world!", flags=["END_STREAM"]
).serialize(),
b"",
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RemoteProtocolError):
conn.request("GET", "https://example.com/")
with pytest.raises(RemoteProtocolError):
conn.request("GET", "https://example.com/")
async def test_proxy_tunneling_http2():
"""
Send an HTTP/2 request via a proxy.
"""
network_backend = HTTP1ThenHTTP2Backend(
[
# The initial response to the proxy CONNECT
b"HTTP/1.1 200 OK\r\n\r\n",
# The actual response from the remote server
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
], )
async with AsyncHTTPProxy(
proxy_url="http://localhost:8080/",
network_backend=network_backend,
http2=True,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
async with proxy.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncTunnelHTTPConnection ['https://example.com:443', HTTP/2, ACTIVE, Request Count: 1]>"
]
await response.aread()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncTunnelHTTPConnection ['https://example.com:443', HTTP/2, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a tunneled proxy can only handle HTTPS requests to the same origin.
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
async def test_http2_connection_with_rst_stream():
"""
If a stream reset occurs, then no response will be returned,
but the connection will remain reusable for other requests.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
# Stream is closed midway through the first response...
hyperframe.frame.RstStreamFrame(stream_id=1, error_code=8).serialize(),
# ...Which doesn't prevent the second response.
hyperframe.frame.HeadersFrame(
stream_id=3,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=3,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
b"",
])
async with AsyncHTTP2Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RemoteProtocolError):
await conn.request("GET", "https://example.com/")
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
def process_story(story_name, raw_directory, target_directory):
"""
Processes and builds the appropriate output for a given raw story.
"""
print("Processing {}...".format(story_name), end='')
raw_story = os.path.join(raw_directory, story_name)
target_story = os.path.join(target_directory, story_name)
with codecs.open(raw_story, encoding="utf-8") as f:
story_json = json.load(f)
cases = story_json["cases"]
seqno = 0
output_data = {
"cases": [],
"description":
("Encoded headers produced by the Python HPACK library, "
"version {}".format(hpack.__version__)),
}
encoder = hpack.Encoder()
for case in cases:
# First we need to resize the header table.
table_size = case.get("header_table_size")
if table_size is not None:
encoder.header_table_size = table_size
headers = headers_from_story(case["headers"])
result = binascii.hexlify(encoder.encode(headers)).decode('ascii')
# Provide the result
output_case = {
"seqno": seqno,
"wire": result,
"headers": case["headers"],
}
if table_size is not None:
output_case["header_table_size"] = table_size
output_data["cases"].append(output_case)
seqno += 1
with codecs.open(target_story, mode="wb", encoding="utf-8") as f:
json.dump(output_data, f, indent=2, sort_keys=True)
print("complete.")
async def http2(reader, writer):
addr = writer.get_extra_info("peername")
print("OPEN:", addr)
stream = await http2_StreamWrapper(reader, writer)
if (PREFACE != await stream.read(preface_length)):
stream.conn.Close()
return
hpack_decoder = hpack.Decoder()
hpack_encoder = hpack.Encoder()
# Length(24) Type(8) flags(8) [R(1) StreamIdentifier(31)] FramePayload(Length)
while (True):
Length = int.from_bytes(await stream.read(3), "big")
if (Length == 0):
stream.conn.Close()
return
Type = await stream.read(1)
flags = int.from_bytes(await stream.read(1), "big")
RnS = await stream.read(4)
#print(http2_frame.bytes_frame_name[Type])
FramePayload = await stream.read(Length)
Payload = http2_parser.frame_payload_parsers[Type](FramePayload, flags)
frame_type = http2_frame.bytes_frame_name[Type]
stream_identifire = int.from_bytes(RnS, "big")
if (frame_type is "HEADERS"):
content, content_type, status = b"", "", 404
request = {}
for p in hpack_decoder.decode(Payload[-1]):
request[p[0]] = p[1]
if (request[":method"] == "GET"):
if (request[":path"].find("?") != -1):
print("GEPT...>>> ", request[":path"])
pfg = request[":path"].split("?")
content, content_type, status = gept.gept(
pfg[0], pfg[1].encode("utf-8"),
message_dir + "/412.html")
else:
print("GET...>>> ", request[":path"])
content, content_type, status = get.get(
req=request[":path"],
root_dir=root_dir,
message_dir=message_dir)
elif (request[":method"] == "POST"):
print("POST--->:", request[":path"])
post_data = b""
print("reading ...")
if (not (flags & 0x01)):
while True:
Length = int.from_bytes(await stream.read(3), "big")
Type = await stream.read(1)
flags = int.from_bytes(await stream.read(1), "big")
RnS = await stream.read(4)
FramePayload = await stream.read(Length)
if (Type == b"\x00"):
Payload = http2_parser.frame_payload_parsers[Type](
FramePayload, flags)
post_data += Payload[0]
if (flags & 0x01):
break
print(len(post_data))
content, content_type, status = post.post(
request[":path"], post_data, message_dir)
rh = [(":status", status), ("server", SERVER_NAME),
("content-type", content_type),
("content-length", len(content)),
("x-frame-options", "SAMEORIGIN"),
("x-xss-protection", "1; mode=block"),
("cache-control", "no-cache")]
res_header_frame = Http2Frame()
res_header_frame.SetFrameType(
http2_frame.frame_type_name["HEADERS"])
res_header_frame.SetStreamIdentifire(stream_identifire)
res_header_frame.SetFramePayload(hpack_encoder.encode(rh))
res_header_frame.SetFlags([http2_frame.frame_flag["END_HEADERS"]])
await stream.conn.Send(res_header_frame.Gen())
res_data = parser.bytes_parseq(content, default_max_frame_size)
res_header_frame.SetFlags([http2_frame.frame_flag["NONE"]])
res_data_frame = res_header_frame
res_data_frame.SetFrameType(http2_frame.frame_type_name["DATA"])
if (len(res_data) == 1):
res_data_frame.SetFramePayload(res_data[0])
res_data_frame.SetFlags([http2_frame.frame_flag["END_STREAM"]])
await stream.conn.Send(res_data_frame.Gen())
for d in res_data[:-1]:
res_data_frame.SetFramePayload(d)
await stream.conn.Send(res_data_frame.Gen())
res_data_frame.SetFramePayload(res_data[-1])
res_data_frame.SetFlags([http2_frame.frame_flag["END_STREAM"]])
await stream.conn.Send(res_data_frame.Gen())
elif (frame_type is "SETTINGS"):
print(Payload)
if (flags == b"\x01"):
break
hpack_encoder.header_table_size = int.from_bytes(
Payload[0][1], "big")
res_settings_frame = Http2Frame()
res_settings_frame.SetFrameType(
http2_frame.frame_type_name["SETTINGS"])
res_settings_frame.SetStreamIdentifire(stream_identifire)
res_settings_frame.SetFlags([http2_frame.frame_flag["NONE"]])
res_settings_frame.SetFramePayload(
b"\x00\x01\x00\x02\x00\x00\x00\x03\x00\x00\x03\xe8\x00\x04\x00\xff\xff\xff\x00\x05\x00\xff\xff\xff"
)
await stream.conn.Send(res_settings_frame.Gen())
wuf = Http2Frame()
wuf.SetFrameType(http2_frame.frame_type_name["WINDOW_UPDATE"])
wuf.SetStreamIdentifire(0)
wuf.SetFramePayload(b"\x00\xff\xff\xff")
wuf.SetFlags([http2_frame.frame_flag["NONE"]])
await stream.conn.Send(wuf.Gen())
elif (frame_type is "WINDOW_UPDATE"):
print(Payload)
elif (frame_type is "RST_STREAM"):
print(Payload)
stream.conn.Close()
elif (frame_type is "DATA"):
#print(Payload)
pass
elif (frame_type is "PING"):
print("PING")
ping_frame = Http2Frame()
ping_frame.SetFrameType(http2_frame.frame_type_name["PING"])
ping_frame.SetStreamIdentifire(stream_identifire)
ping_frame.SetFramePayload(Payload)
ping_frame.SetFlags([http2_frame.frame_flag["ACK"]])
await stream.conn.Send(ping_frame.Gen())
def read_frame(sock):
'''Read exactly 1 h2 frame from socket '''
data = read_bytes(sock, 9)
frame, payload_len = hyperframe.frame.Frame.parse_frame_header(data)
if payload_len:
data = read_bytes(sock, payload_len)
frame.parse_body(memoryview(data))
return frame
if __name__ == '__main__':
# connect
hostname = 'www.google.com'
hpack_enc = hpack.Encoder()
hpack_dec = hpack.Decoder()
sock = connect(hostname)
# send connection preface
write(sock, MAGIC)
write(sock, hyperframe.frame.SettingsFrame())
# send request
headers = hyperframe.frame.HeadersFrame(3)
headers.data = hpack_enc.encode([
(b':method', b'GET'),
(b':scheme', b'https'),
(b':authority', b'www.google.com'),
(b':path', b'/'),
])
def __init__(self, options, is_request, params):
BaseProcessor.__init__(self, options, is_request, params)
self.compressor = hpack.Encoder()
self.decompressor = hpack.Decoder()
self.sensitive = []
def encode_payload(data):
encoder = hpack.Encoder()
data_encoded = encoder.encode(data)
return data_encoded