async def test_proxy_tunneling_http2():
"""
Send an HTTP/2 request via a proxy.
"""
network_backend = HTTP1ThenHTTP2Backend(
[
# The initial response to the proxy CONNECT
b"HTTP/1.1 200 OK\r\n\r\n",
# The actual response from the remote server
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
], )
async with AsyncHTTPProxy(
proxy_url="http://localhost:8080/",
network_backend=network_backend,
http2=True,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
async with proxy.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncTunnelHTTPConnection ['https://example.com:443', HTTP/2, ACTIVE, Request Count: 1]>"
]
await response.aread()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncTunnelHTTPConnection ['https://example.com:443', HTTP/2, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a tunneled proxy can only handle HTTPS requests to the same origin.
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
async def test_socks5_request():
"""
Send an HTTP request via a SOCKS proxy.
"""
network_backend = AsyncMockBackend([
# The initial socks CONNECT
# v5 NOAUTH
b"\x05\x00",
# v5 SUC RSV IP4 127 .0 .0 .1 :80
b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50",
# The actual response from the remote server
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncSOCKSProxy(
proxy_url="socks5://localhost:8080/",
network_backend=network_backend,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
async with proxy.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncSocks5Connection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
await response.aread()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncSocks5Connection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a tunneled proxy can only handle HTTPS requests to the same origin.
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
async def test_http2_connection():
origin = Origin(b"https", b"example.com", 443)
network_backend = AsyncMockBackend(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
],
http2=True,
)
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend,
http2=True) as conn:
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
assert response.extensions["http_version"] == b"HTTP/2"
async def test_http2_connection():
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
])
async with AsyncHTTP2Connection(origin=origin,
stream=stream,
keepalive_expiry=5.0) as conn:
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
assert conn.is_idle()
assert conn.is_available()
assert not conn.is_closed()
assert not conn.has_expired()
assert (conn.info() ==
"'https://example.com:443', HTTP/2, IDLE, Request Count: 1")
assert (
repr(conn) ==
"<AsyncHTTP2Connection ['https://example.com:443', IDLE, Request Count: 1]>"
)
async def test_http11_connection_with_local_protocol_error():
"""
If a local protocol error occurs, then no response will be returned,
and the connection will not be reusable.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTP11Connection(origin=origin, stream=stream) as conn:
with pytest.raises(LocalProtocolError) as exc_info:
await conn.request("GET",
"https://example.com/",
headers={"Host": "\0"})
assert str(exc_info.value) == "Illegal header value b'\\x00'"
assert not conn.is_idle()
assert conn.is_closed()
assert not conn.is_available()
assert not conn.has_expired()
assert (
repr(conn) ==
"<AsyncHTTP11Connection ['https://example.com:443', CLOSED, Request Count: 1]>"
)
def test_http2_connection_attempt_close():
"""
A connection can only be closed when it is idle.
"""
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=1, data=b"Hello, world!", flags=["END_STREAM"]
).serialize(),
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
with conn.stream("GET", "https://example.com/") as response:
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
conn.close()
with pytest.raises(ConnectionNotAvailable):
conn.request("GET", "https://example.com/")
async def test_http11_connection_unread_response():
"""
If the client releases the response without reading it to termination,
then the connection will not be reusable.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTP11Connection(origin=origin, stream=stream) as conn:
async with conn.stream("GET", "https://example.com/") as response:
assert response.status == 200
assert not conn.is_idle()
assert conn.is_closed()
assert not conn.is_available()
assert not conn.has_expired()
assert (
repr(conn) ==
"<AsyncHTTP11Connection ['https://example.com:443', CLOSED, Request Count: 1]>"
)
def test_http2_connection_post_request():
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=1, data=b"Hello, world!", flags=["END_STREAM"]
).serialize(),
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
response = conn.request(
"POST",
"https://example.com/",
headers={b"content-length": b"17"},
content=b'{"data": "upload"}',
)
assert response.status == 200
assert response.content == b"Hello, world!"
def test_proxy_tunneling():
"""
Send an HTTPS request via a proxy.
"""
network_backend = MockBackend([
# The initial response to the proxy CONNECT
b"HTTP/1.1 200 OK\r\n\r\n",
# The actual response from the remote server
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTPProxy(
proxy_url="http://localhost:8080/",
network_backend=network_backend,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
with proxy.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<TunnelHTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<TunnelHTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a tunneled proxy can only handle HTTPS requests to the same origin.
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
def test_http2_connection_with_flow_control():
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
# Available flow: 65,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 75,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 85,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 95,535
hyperframe.frame.WindowUpdateFrame(
stream_id=0, window_increment=10_000
).serialize(),
hyperframe.frame.WindowUpdateFrame(
stream_id=1, window_increment=10_000
).serialize(),
# Available flow: 105,535
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=1, data=b"100,000 bytes received", flags=["END_STREAM"]
).serialize(),
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
response = conn.request(
"POST",
"https://example.com/",
content=b"x" * 100_000,
)
assert response.status == 200
assert response.content == b"100,000 bytes received"
def test_http2_request_to_incorrect_origin():
"""
A connection can only send requests to whichever origin it is connected to.
"""
origin = Origin(b"https", b"example.com", 443)
stream = MockStream([])
with HTTP2Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RuntimeError):
conn.request("GET", "https://other.com/")
async def test_proxy_forwarding():
"""
Send an HTTP request via a proxy.
"""
network_backend = AsyncMockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTPProxy(
proxy_url="http://localhost:8080/",
max_connections=10,
network_backend=network_backend,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
async with proxy.stream("GET", "http://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncForwardHTTPConnection ['http://localhost:8080', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
await response.aread()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<AsyncForwardHTTPConnection ['http://localhost:8080', HTTP/1.1, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a forwarding proxy can handle HTTP requests to any host.
assert proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
async def test_request_to_incorrect_origin():
"""
A connection can only send requests whichever origin it is connected to.
"""
origin = Origin(b"https", b"example.com", 443)
network_backend = AsyncMockBackend([])
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend) as conn:
with pytest.raises(RuntimeError):
await conn.request("GET", "https://other.com/")
def test_http2_connection_with_remote_protocol_error():
"""
If a remote protocol error occurs, then no response will be returned,
and the connection will not be reusable.
"""
origin = Origin(b"https", b"example.com", 443)
stream = MockStream([b"Wait, this isn't valid HTTP!", b""])
with HTTP2Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RemoteProtocolError):
conn.request("GET", "https://example.com/")
async def test_http11_connection_attempt_close():
"""
A connection can only be closed when it is idle.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTP11Connection(origin=origin, stream=stream) as conn:
async with conn.stream("GET", "https://example.com/") as response:
await response.aread()
assert response.status == 200
assert response.content == b"Hello, world!"
async def test_http11_connection_handles_one_active_request():
"""
Attempting to send a request while one is already in-flight will raise
a ConnectionNotAvailable exception.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTP11Connection(origin=origin, stream=stream) as conn:
async with conn.stream("GET", "https://example.com/"):
with pytest.raises(ConnectionNotAvailable):
await conn.request("GET", "https://example.com/")
async def test_uds_connections():
# We're not actually testing Unix Domain Sockets here, because we're just
# using a mock backend, but at least we're covering the UDS codepath
# in `connection.py` which we may as well do.
origin = Origin(b"https", b"example.com", 443)
network_backend = AsyncMockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend,
uds="/mock/example") as conn:
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
async def test_http11_connection_with_remote_protocol_error():
"""
If a remote protocol error occurs, then no response will be returned,
and the connection will not be reusable.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([b"Wait, this isn't valid HTTP!", b""])
async with AsyncHTTP11Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RemoteProtocolError):
await conn.request("GET", "https://example.com/")
assert not conn.is_idle()
assert conn.is_closed()
assert not conn.is_available()
assert not conn.has_expired()
assert (
repr(conn) ==
"<AsyncHTTP11Connection ['https://example.com:443', CLOSED, Request Count: 1]>"
)
def test_http2_connection_with_goaway():
"""
If a stream reset occurs, then no response will be returned,
but the connection will remain reusable for other requests.
"""
origin = Origin(b"https", b"example.com", 443)
stream = MockStream(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
# Connection is closed midway through the first response...
hyperframe.frame.GoAwayFrame(stream_id=0, error_code=0).serialize(),
# ...We'll never get to this second response.
hyperframe.frame.HeadersFrame(
stream_id=3,
data=hpack.Encoder().encode(
[
(b":status", b"200"),
(b"content-type", b"plain/text"),
]
),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(
stream_id=3, data=b"Hello, world!", flags=["END_STREAM"]
).serialize(),
b"",
]
)
with HTTP2Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RemoteProtocolError):
conn.request("GET", "https://example.com/")
with pytest.raises(RemoteProtocolError):
conn.request("GET", "https://example.com/")
async def test_concurrent_requests_not_available_on_http11_connections():
"""
Attempting to issue a request against an already active HTTP/1.1 connection
will raise a `ConnectionNotAvailable` exception.
"""
origin = Origin(b"https", b"example.com", 443)
network_backend = AsyncMockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend,
keepalive_expiry=5.0) as conn:
async with conn.stream("GET", "https://example.com/"):
with pytest.raises(ConnectionNotAvailable):
await conn.request("GET", "https://example.com/")
def test_http11_connection():
origin = Origin(b"https", b"example.com", 443)
stream = MockStream([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTP11Connection(origin=origin, stream=stream,
keepalive_expiry=5.0) as conn:
response = conn.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
assert conn.is_idle()
assert not conn.is_closed()
assert conn.is_available()
assert not conn.has_expired()
assert (
repr(conn) ==
"<HTTP11Connection ['https://example.com:443', IDLE, Request Count: 1]>"
)
async def test_http_connection():
origin = Origin(b"https", b"example.com", 443)
network_backend = AsyncMockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend,
keepalive_expiry=5.0) as conn:
assert not conn.is_idle()
assert not conn.is_closed()
assert not conn.is_available()
assert not conn.has_expired()
assert repr(conn) == "<AsyncHTTPConnection [CONNECTING]>"
async with conn.stream("GET", "https://example.com/") as response:
assert (
repr(conn) ==
"<AsyncHTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
)
await response.aread()
assert response.status == 200
assert response.content == b"Hello, world!"
assert conn.is_idle()
assert not conn.is_closed()
assert conn.is_available()
assert not conn.has_expired()
assert (
repr(conn) ==
"<AsyncHTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
)
async def test_http2_connection_with_rst_stream():
"""
If a stream reset occurs, then no response will be returned,
but the connection will remain reusable for other requests.
"""
origin = Origin(b"https", b"example.com", 443)
stream = AsyncMockStream([
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
# Stream is closed midway through the first response...
hyperframe.frame.RstStreamFrame(stream_id=1, error_code=8).serialize(),
# ...Which doesn't prevent the second response.
hyperframe.frame.HeadersFrame(
stream_id=3,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=3,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
b"",
])
async with AsyncHTTP2Connection(origin=origin, stream=stream) as conn:
with pytest.raises(RemoteProtocolError):
await conn.request("GET", "https://example.com/")
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
async def test_connection_retries():
origin = Origin(b"https", b"example.com", 443)
content = [
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
]
network_backend = NeedsRetryBackend(content)
async with AsyncHTTPConnection(origin=origin,
network_backend=network_backend,
retries=3) as conn:
response = await conn.request("GET", "https://example.com/")
assert response.status == 200
network_backend = NeedsRetryBackend(content)
async with AsyncHTTPConnection(
origin=origin,
network_backend=network_backend,
) as conn:
with pytest.raises(ConnectError):
await conn.request("GET", "https://example.com/")