def test_http2_connection():
origin = Origin(b"https", b"example.com", 443)
network_backend = MockBackend(
[
hyperframe.frame.SettingsFrame().serialize(),
hyperframe.frame.HeadersFrame(
stream_id=1,
data=hpack.Encoder().encode([
(b":status", b"200"),
(b"content-type", b"plain/text"),
]),
flags=["END_HEADERS"],
).serialize(),
hyperframe.frame.DataFrame(stream_id=1,
data=b"Hello, world!",
flags=["END_STREAM"]).serialize(),
],
http2=True,
)
with HTTPConnection(origin=origin,
network_backend=network_backend,
http2=True) as conn:
response = conn.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
assert response.extensions["http_version"] == b"HTTP/2"
def test_connection_pool_with_no_keepalive_connections_allowed():
"""
When 'max_keepalive_connections=0' is used, IDLE connections should not
be returned to the pool.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with ConnectionPool(max_keepalive_connections=0,
network_backend=network_backend) as pool:
# Sending an intial request, which once complete will not return to the pool.
with pool.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in pool.connections]
assert info == []
def test_connection_pool_with_immediate_expiry():
"""
Connection pools with keepalive_expiry=0.0 should immediately expire
keep alive connections.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with ConnectionPool(
keepalive_expiry=0.0,
network_backend=network_backend,
) as pool:
# Sending an intial request, which once complete will not return to the pool.
with pool.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in pool.connections]
assert info == []
def test_connection_pool_with_close():
"""
HTTP/1.1 requests that include a 'Connection: Close' header should
not be returned to the connection pool.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with ConnectionPool(network_backend=network_backend) as pool:
# Sending an intial request, which once complete will not return to the pool.
with pool.stream("GET",
"https://example.com/",
headers={"Connection": "close"}) as response:
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in pool.connections]
assert info == []
def test_proxy_tunneling_with_auth():
"""
Send an authenticated HTTPS request via a proxy.
"""
network_backend = MockBackend([
# The initial response to the proxy CONNECT
b"HTTP/1.1 200 OK\r\n\r\n",
# The actual response from the remote server
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTPProxy(
proxy_url="http://localhost:8080/",
proxy_auth=("username", "password"),
network_backend=network_backend,
) as proxy:
response = proxy.request("GET", "https://example.com/")
assert response.status == 200
assert response.content == b"Hello, world!"
# Dig into this private property as a cheap lazy way of
# checking that the proxy header is set correctly.
assert proxy._proxy_headers == [ # type: ignore
(b"Proxy-Authorization", b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=")
]
def test_request_to_incorrect_origin():
"""
A connection can only send requests whichever origin it is connected to.
"""
origin = Origin(b"https", b"example.com", 443)
network_backend = MockBackend([])
with HTTPConnection(origin=origin,
network_backend=network_backend) as conn:
with pytest.raises(RuntimeError):
conn.request("GET", "https://other.com/")
def test_socks5_request():
"""
Send an HTTP request via a SOCKS proxy.
"""
network_backend = MockBackend([
# The initial socks CONNECT
# v5 NOAUTH
b"\x05\x00",
# v5 SUC RSV IP4 127 .0 .0 .1 :80
b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50",
# The actual response from the remote server
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with SOCKSProxy(
proxy_url="socks5://localhost:8080/",
network_backend=network_backend,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
with proxy.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<Socks5Connection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<Socks5Connection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a tunneled proxy can only handle HTTPS requests to the same origin.
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
def test_uds_connections():
# We're not actually testing Unix Domain Sockets here, because we're just
# using a mock backend, but at least we're covering the UDS codepath
# in `connection.py` which we may as well do.
origin = Origin(b"https", b"example.com", 443)
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTPConnection(origin=origin,
network_backend=network_backend,
uds="/mock/example") as conn:
response = conn.request("GET", "https://example.com/")
assert response.status == 200
def test_proxy_tunneling_with_403():
"""
Send an HTTPS request via a proxy.
"""
network_backend = MockBackend([
b"HTTP/1.1 403 Permission Denied\r\n"
b"\r\n",
])
with HTTPProxy(
proxy_url="http://localhost:8080/",
network_backend=network_backend,
) as proxy:
with pytest.raises(ProxyError) as exc_info:
proxy.request("GET", "https://example.com/")
assert str(exc_info.value) == "403 Permission Denied"
assert not proxy.connections
def test_proxy_forwarding():
"""
Send an HTTP request via a proxy.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTPProxy(
proxy_url="http://localhost:8080/",
max_connections=10,
network_backend=network_backend,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
with proxy.stream("GET", "http://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<ForwardHTTPConnection ['http://localhost:8080', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<ForwardHTTPConnection ['http://localhost:8080', HTTP/1.1, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
# A connection on a forwarding proxy can handle HTTP requests to any host.
assert proxy.connections[0].can_handle_request(
Origin(b"http", b"example.com", 80))
assert proxy.connections[0].can_handle_request(
Origin(b"http", b"other.com", 80))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"example.com", 443))
assert not proxy.connections[0].can_handle_request(
Origin(b"https", b"other.com", 443))
def test_authenticated_socks5_request():
"""
Send an HTTP request via a SOCKS proxy.
"""
network_backend = MockBackend([
# The initial socks CONNECT
# v5 USERNAME/PASSWORD
b"\x05\x02",
# v1 VALID USERNAME/PASSWORD
b"\x01\x00",
# v5 SUC RSV IP4 127 .0 .0 .1 :80
b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50",
# The actual response from the remote server
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with SOCKSProxy(
proxy_url="socks5://localhost:8080/",
proxy_auth=(b"username", b"password"),
network_backend=network_backend,
) as proxy:
# Sending an intial request, which once complete will return to the pool, IDLE.
with proxy.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in proxy.connections]
assert info == [
"<Socks5Connection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in proxy.connections]
assert info == [
"<Socks5Connection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
]
assert proxy.connections[0].is_idle()
assert proxy.connections[0].is_available()
assert not proxy.connections[0].is_closed()
def test_concurrent_requests_not_available_on_http11_connections():
"""
Attempting to issue a request against an already active HTTP/1.1 connection
will raise a `ConnectionNotAvailable` exception.
"""
origin = Origin(b"https", b"example.com", 443)
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTPConnection(origin=origin,
network_backend=network_backend,
keepalive_expiry=5.0) as conn:
with conn.stream("GET", "https://example.com/"):
with pytest.raises(ConnectionNotAvailable):
conn.request("GET", "https://example.com/")
def test_trace_request():
"""
The 'trace' request extension allows for a callback function to inspect the
internal events that occur while sending a request.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
called = []
def trace(name, kwargs):
called.append(name)
with ConnectionPool(network_backend=network_backend) as pool:
pool.request("GET",
"https://example.com/",
extensions={"trace": trace})
assert called == [
"connection.connect_tcp.started",
"connection.connect_tcp.complete",
"connection.start_tls.started",
"connection.start_tls.complete",
"http11.send_request_headers.started",
"http11.send_request_headers.complete",
"http11.send_request_body.started",
"http11.send_request_body.complete",
"http11.receive_response_headers.started",
"http11.receive_response_headers.complete",
"http11.receive_response_body.started",
"http11.receive_response_body.complete",
"http11.response_closed.started",
"http11.response_closed.complete",
]
def test_connection_pool_concurrency():
"""
HTTP/1.1 requests made in concurrency must not ever exceed the maximum number
of allowable connection in the pool.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
def fetch(pool, domain, info_list):
with pool.stream("GET", f"http://{domain}/") as response:
info = [repr(c) for c in pool.connections]
info_list.append(info)
response.read()
with ConnectionPool(max_connections=1,
network_backend=network_backend) as pool:
info_list: List[str] = []
with concurrency.open_nursery() as nursery:
for domain in ["a.com", "b.com", "c.com", "d.com", "e.com"]:
nursery.start_soon(fetch, pool, domain, info_list)
for item in info_list:
# Check that each time we inspected the connection pool, only a
# single connection was established at any one time.
assert len(item) == 1
# Each connection was to a different host, and only sent a single
# request on that connection.
assert item[0] in [
"<HTTPConnection ['http://a.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>",
"<HTTPConnection ['http://b.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>",
"<HTTPConnection ['http://c.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>",
"<HTTPConnection ['http://d.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>",
"<HTTPConnection ['http://e.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>",
]
def test_connection_pool_concurrency_same_domain_keepalive():
"""
HTTP/1.1 requests made in concurrency must not ever exceed the maximum number
of allowable connection in the pool.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
] * 5)
def fetch(pool, domain, info_list):
with pool.stream("GET", f"https://{domain}/") as response:
info = [repr(c) for c in pool.connections]
info_list.append(info)
response.read()
with ConnectionPool(max_connections=1,
network_backend=network_backend,
http2=True) as pool:
info_list: List[str] = []
with concurrency.open_nursery() as nursery:
for domain in ["a.com", "a.com", "a.com", "a.com", "a.com"]:
nursery.start_soon(fetch, pool, domain, info_list)
for item in info_list:
# Check that each time we inspected the connection pool, only a
# single connection was established at any one time.
assert len(item) == 1
# The connection sent multiple requests.
assert item[0] in [
"<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>",
"<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 2]>",
"<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 3]>",
"<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 4]>",
"<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 5]>",
]
def test_socks5_request_incorrect_auth():
"""
Attempt to send an HTTP request via an authenticated SOCKS proxy,
wit incorrect authentication credentials.
"""
network_backend = MockBackend([
# v5 USERNAME/PASSWORD
b"\x05\x02",
# v1 INVALID USERNAME/PASSWORD
b"\x01\x01",
])
with SOCKSProxy(
proxy_url="socks5://localhost:8080/",
proxy_auth=(b"invalid", b"invalid"),
network_backend=network_backend,
) as proxy:
# Sending a request, which the proxy rejects
with pytest.raises(ProxyError) as exc_info:
proxy.request("GET", "https://example.com/")
assert str(exc_info.value) == "Invalid username/password"
assert not proxy.connections
def test_socks5_request_failed_to_provide_auth():
"""
Attempt to send an HTTP request via an authenticated SOCKS proxy,
without providing authentication credentials.
"""
network_backend = MockBackend([
# v5 USERNAME/PASSWORD
b"\x05\x02",
])
with SOCKSProxy(
proxy_url="socks5://localhost:8080/",
network_backend=network_backend,
) as proxy:
# Sending a request, which the proxy rejects
with pytest.raises(ProxyError) as exc_info:
proxy.request("GET", "https://example.com/")
assert (
str(exc_info.value) ==
"Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD."
)
assert not proxy.connections
def test_socks5_request_connect_failed():
"""
Attempt to send an HTTP request via a SOCKS proxy, resulting in a connect failure.
"""
network_backend = MockBackend([
# The initial socks CONNECT
# v5 NOAUTH
b"\x05\x00",
# v5 NO RSV IP4 0 .0 .0 .0 :00
b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00",
])
with SOCKSProxy(
proxy_url="socks5://localhost:8080/",
network_backend=network_backend,
) as proxy:
# Sending a request, which the proxy rejects
with pytest.raises(ProxyError) as exc_info:
proxy.request("GET", "https://example.com/")
assert (str(exc_info.value) ==
"Proxy Server could not connect: Connection refused.")
assert not proxy.connections
def test_http_connection():
origin = Origin(b"https", b"example.com", 443)
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with HTTPConnection(origin=origin,
network_backend=network_backend,
keepalive_expiry=5.0) as conn:
assert not conn.is_idle()
assert not conn.is_closed()
assert not conn.is_available()
assert not conn.has_expired()
assert repr(conn) == "<HTTPConnection [CONNECTING]>"
with conn.stream("GET", "https://example.com/") as response:
assert (
repr(conn) ==
"<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
)
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
assert conn.is_idle()
assert not conn.is_closed()
assert conn.is_available()
assert not conn.has_expired()
assert (
repr(conn) ==
"<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
)
def test_connection_pool_with_http_exception():
"""
HTTP/1.1 requests that result in an exception during the connection should
not be returned to the connection pool.
"""
network_backend = MockBackend([b"Wait, this isn't valid HTTP!"])
called = []
def trace(name, kwargs):
called.append(name)
with ConnectionPool(network_backend=network_backend) as pool:
# Sending an initial request, which once complete will not return to the pool.
with pytest.raises(Exception):
pool.request("GET",
"https://example.com/",
extensions={"trace": trace})
info = [repr(c) for c in pool.connections]
assert info == []
assert called == [
"connection.connect_tcp.started",
"connection.connect_tcp.complete",
"connection.start_tls.started",
"connection.start_tls.complete",
"http11.send_request_headers.started",
"http11.send_request_headers.complete",
"http11.send_request_body.started",
"http11.send_request_body.complete",
"http11.receive_response_headers.started",
"http11.receive_response_headers.failed",
"http11.response_closed.started",
"http11.response_closed.complete",
]
def test_connection_pool_with_keepalive():
"""
By default HTTP/1.1 requests should be returned to the connection pool.
"""
network_backend = MockBackend([
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
b"HTTP/1.1 200 OK\r\n",
b"Content-Type: plain/text\r\n",
b"Content-Length: 13\r\n",
b"\r\n",
b"Hello, world!",
])
with ConnectionPool(network_backend=network_backend, ) as pool:
# Sending an intial request, which once complete will return to the pool, IDLE.
with pool.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>"
]
# Sending a second request to the same origin will reuse the existing IDLE connection.
with pool.stream("GET", "https://example.com/") as response:
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 2]>"
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 2]>"
]
# Sending a request to a different origin will not reuse the existing IDLE connection.
with pool.stream("GET", "http://example.com/") as response:
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['http://example.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>",
"<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 2]>",
]
response.read()
assert response.status == 200
assert response.content == b"Hello, world!"
info = [repr(c) for c in pool.connections]
assert info == [
"<HTTPConnection ['http://example.com:80', HTTP/1.1, IDLE, Request Count: 1]>",
"<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 2]>",
]