def test_http2_connection(): origin = Origin(b"https", b"example.com", 443) network_backend = MockBackend( [ hyperframe.frame.SettingsFrame().serialize(), hyperframe.frame.HeadersFrame( stream_id=1, data=hpack.Encoder().encode([ (b":status", b"200"), (b"content-type", b"plain/text"), ]), flags=["END_HEADERS"], ).serialize(), hyperframe.frame.DataFrame(stream_id=1, data=b"Hello, world!", flags=["END_STREAM"]).serialize(), ], http2=True, ) with HTTPConnection(origin=origin, network_backend=network_backend, http2=True) as conn: response = conn.request("GET", "https://example.com/") assert response.status == 200 assert response.content == b"Hello, world!" assert response.extensions["http_version"] == b"HTTP/2"
def test_connection_pool_with_no_keepalive_connections_allowed(): """ When 'max_keepalive_connections=0' is used, IDLE connections should not be returned to the pool. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with ConnectionPool(max_keepalive_connections=0, network_backend=network_backend) as pool: # Sending an intial request, which once complete will not return to the pool. with pool.stream("GET", "https://example.com/") as response: info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in pool.connections] assert info == []
def test_connection_pool_with_immediate_expiry(): """ Connection pools with keepalive_expiry=0.0 should immediately expire keep alive connections. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with ConnectionPool( keepalive_expiry=0.0, network_backend=network_backend, ) as pool: # Sending an intial request, which once complete will not return to the pool. with pool.stream("GET", "https://example.com/") as response: info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in pool.connections] assert info == []
def test_connection_pool_with_close(): """ HTTP/1.1 requests that include a 'Connection: Close' header should not be returned to the connection pool. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with ConnectionPool(network_backend=network_backend) as pool: # Sending an intial request, which once complete will not return to the pool. with pool.stream("GET", "https://example.com/", headers={"Connection": "close"}) as response: info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in pool.connections] assert info == []
def test_proxy_tunneling_with_auth(): """ Send an authenticated HTTPS request via a proxy. """ network_backend = MockBackend([ # The initial response to the proxy CONNECT b"HTTP/1.1 200 OK\r\n\r\n", # The actual response from the remote server b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with HTTPProxy( proxy_url="http://localhost:8080/", proxy_auth=("username", "password"), network_backend=network_backend, ) as proxy: response = proxy.request("GET", "https://example.com/") assert response.status == 200 assert response.content == b"Hello, world!" # Dig into this private property as a cheap lazy way of # checking that the proxy header is set correctly. assert proxy._proxy_headers == [ # type: ignore (b"Proxy-Authorization", b"Basic dXNlcm5hbWU6cGFzc3dvcmQ=") ]
def test_request_to_incorrect_origin(): """ A connection can only send requests whichever origin it is connected to. """ origin = Origin(b"https", b"example.com", 443) network_backend = MockBackend([]) with HTTPConnection(origin=origin, network_backend=network_backend) as conn: with pytest.raises(RuntimeError): conn.request("GET", "https://other.com/")
def test_socks5_request(): """ Send an HTTP request via a SOCKS proxy. """ network_backend = MockBackend([ # The initial socks CONNECT # v5 NOAUTH b"\x05\x00", # v5 SUC RSV IP4 127 .0 .0 .1 :80 b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", # The actual response from the remote server b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with SOCKSProxy( proxy_url="socks5://localhost:8080/", network_backend=network_backend, ) as proxy: # Sending an intial request, which once complete will return to the pool, IDLE. with proxy.stream("GET", "https://example.com/") as response: info = [repr(c) for c in proxy.connections] assert info == [ "<Socks5Connection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in proxy.connections] assert info == [ "<Socks5Connection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>" ] assert proxy.connections[0].is_idle() assert proxy.connections[0].is_available() assert not proxy.connections[0].is_closed() # A connection on a tunneled proxy can only handle HTTPS requests to the same origin. assert not proxy.connections[0].can_handle_request( Origin(b"http", b"example.com", 80)) assert not proxy.connections[0].can_handle_request( Origin(b"http", b"other.com", 80)) assert proxy.connections[0].can_handle_request( Origin(b"https", b"example.com", 443)) assert not proxy.connections[0].can_handle_request( Origin(b"https", b"other.com", 443))
def test_uds_connections(): # We're not actually testing Unix Domain Sockets here, because we're just # using a mock backend, but at least we're covering the UDS codepath # in `connection.py` which we may as well do. origin = Origin(b"https", b"example.com", 443) network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with HTTPConnection(origin=origin, network_backend=network_backend, uds="/mock/example") as conn: response = conn.request("GET", "https://example.com/") assert response.status == 200
def test_proxy_tunneling_with_403(): """ Send an HTTPS request via a proxy. """ network_backend = MockBackend([ b"HTTP/1.1 403 Permission Denied\r\n" b"\r\n", ]) with HTTPProxy( proxy_url="http://localhost:8080/", network_backend=network_backend, ) as proxy: with pytest.raises(ProxyError) as exc_info: proxy.request("GET", "https://example.com/") assert str(exc_info.value) == "403 Permission Denied" assert not proxy.connections
def test_proxy_forwarding(): """ Send an HTTP request via a proxy. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with HTTPProxy( proxy_url="http://localhost:8080/", max_connections=10, network_backend=network_backend, ) as proxy: # Sending an intial request, which once complete will return to the pool, IDLE. with proxy.stream("GET", "http://example.com/") as response: info = [repr(c) for c in proxy.connections] assert info == [ "<ForwardHTTPConnection ['http://localhost:8080', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in proxy.connections] assert info == [ "<ForwardHTTPConnection ['http://localhost:8080', HTTP/1.1, IDLE, Request Count: 1]>" ] assert proxy.connections[0].is_idle() assert proxy.connections[0].is_available() assert not proxy.connections[0].is_closed() # A connection on a forwarding proxy can handle HTTP requests to any host. assert proxy.connections[0].can_handle_request( Origin(b"http", b"example.com", 80)) assert proxy.connections[0].can_handle_request( Origin(b"http", b"other.com", 80)) assert not proxy.connections[0].can_handle_request( Origin(b"https", b"example.com", 443)) assert not proxy.connections[0].can_handle_request( Origin(b"https", b"other.com", 443))
def test_authenticated_socks5_request(): """ Send an HTTP request via a SOCKS proxy. """ network_backend = MockBackend([ # The initial socks CONNECT # v5 USERNAME/PASSWORD b"\x05\x02", # v1 VALID USERNAME/PASSWORD b"\x01\x00", # v5 SUC RSV IP4 127 .0 .0 .1 :80 b"\x05\x00\x00\x01\xff\x00\x00\x01\x00\x50", # The actual response from the remote server b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with SOCKSProxy( proxy_url="socks5://localhost:8080/", proxy_auth=(b"username", b"password"), network_backend=network_backend, ) as proxy: # Sending an intial request, which once complete will return to the pool, IDLE. with proxy.stream("GET", "https://example.com/") as response: info = [repr(c) for c in proxy.connections] assert info == [ "<Socks5Connection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in proxy.connections] assert info == [ "<Socks5Connection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>" ] assert proxy.connections[0].is_idle() assert proxy.connections[0].is_available() assert not proxy.connections[0].is_closed()
def test_concurrent_requests_not_available_on_http11_connections(): """ Attempting to issue a request against an already active HTTP/1.1 connection will raise a `ConnectionNotAvailable` exception. """ origin = Origin(b"https", b"example.com", 443) network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with HTTPConnection(origin=origin, network_backend=network_backend, keepalive_expiry=5.0) as conn: with conn.stream("GET", "https://example.com/"): with pytest.raises(ConnectionNotAvailable): conn.request("GET", "https://example.com/")
def test_trace_request(): """ The 'trace' request extension allows for a callback function to inspect the internal events that occur while sending a request. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) called = [] def trace(name, kwargs): called.append(name) with ConnectionPool(network_backend=network_backend) as pool: pool.request("GET", "https://example.com/", extensions={"trace": trace}) assert called == [ "connection.connect_tcp.started", "connection.connect_tcp.complete", "connection.start_tls.started", "connection.start_tls.complete", "http11.send_request_headers.started", "http11.send_request_headers.complete", "http11.send_request_body.started", "http11.send_request_body.complete", "http11.receive_response_headers.started", "http11.receive_response_headers.complete", "http11.receive_response_body.started", "http11.receive_response_body.complete", "http11.response_closed.started", "http11.response_closed.complete", ]
def test_connection_pool_concurrency(): """ HTTP/1.1 requests made in concurrency must not ever exceed the maximum number of allowable connection in the pool. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) def fetch(pool, domain, info_list): with pool.stream("GET", f"http://{domain}/") as response: info = [repr(c) for c in pool.connections] info_list.append(info) response.read() with ConnectionPool(max_connections=1, network_backend=network_backend) as pool: info_list: List[str] = [] with concurrency.open_nursery() as nursery: for domain in ["a.com", "b.com", "c.com", "d.com", "e.com"]: nursery.start_soon(fetch, pool, domain, info_list) for item in info_list: # Check that each time we inspected the connection pool, only a # single connection was established at any one time. assert len(item) == 1 # Each connection was to a different host, and only sent a single # request on that connection. assert item[0] in [ "<HTTPConnection ['http://a.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>", "<HTTPConnection ['http://b.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>", "<HTTPConnection ['http://c.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>", "<HTTPConnection ['http://d.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>", "<HTTPConnection ['http://e.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>", ]
def test_connection_pool_concurrency_same_domain_keepalive(): """ HTTP/1.1 requests made in concurrency must not ever exceed the maximum number of allowable connection in the pool. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ] * 5) def fetch(pool, domain, info_list): with pool.stream("GET", f"https://{domain}/") as response: info = [repr(c) for c in pool.connections] info_list.append(info) response.read() with ConnectionPool(max_connections=1, network_backend=network_backend, http2=True) as pool: info_list: List[str] = [] with concurrency.open_nursery() as nursery: for domain in ["a.com", "a.com", "a.com", "a.com", "a.com"]: nursery.start_soon(fetch, pool, domain, info_list) for item in info_list: # Check that each time we inspected the connection pool, only a # single connection was established at any one time. assert len(item) == 1 # The connection sent multiple requests. assert item[0] in [ "<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>", "<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 2]>", "<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 3]>", "<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 4]>", "<HTTPConnection ['https://a.com:443', HTTP/1.1, ACTIVE, Request Count: 5]>", ]
def test_socks5_request_incorrect_auth(): """ Attempt to send an HTTP request via an authenticated SOCKS proxy, wit incorrect authentication credentials. """ network_backend = MockBackend([ # v5 USERNAME/PASSWORD b"\x05\x02", # v1 INVALID USERNAME/PASSWORD b"\x01\x01", ]) with SOCKSProxy( proxy_url="socks5://localhost:8080/", proxy_auth=(b"invalid", b"invalid"), network_backend=network_backend, ) as proxy: # Sending a request, which the proxy rejects with pytest.raises(ProxyError) as exc_info: proxy.request("GET", "https://example.com/") assert str(exc_info.value) == "Invalid username/password" assert not proxy.connections
def test_socks5_request_failed_to_provide_auth(): """ Attempt to send an HTTP request via an authenticated SOCKS proxy, without providing authentication credentials. """ network_backend = MockBackend([ # v5 USERNAME/PASSWORD b"\x05\x02", ]) with SOCKSProxy( proxy_url="socks5://localhost:8080/", network_backend=network_backend, ) as proxy: # Sending a request, which the proxy rejects with pytest.raises(ProxyError) as exc_info: proxy.request("GET", "https://example.com/") assert ( str(exc_info.value) == "Requested NO AUTHENTICATION REQUIRED from proxy server, but got USERNAME/PASSWORD." ) assert not proxy.connections
def test_socks5_request_connect_failed(): """ Attempt to send an HTTP request via a SOCKS proxy, resulting in a connect failure. """ network_backend = MockBackend([ # The initial socks CONNECT # v5 NOAUTH b"\x05\x00", # v5 NO RSV IP4 0 .0 .0 .0 :00 b"\x05\x05\x00\x01\x00\x00\x00\x00\x00\x00", ]) with SOCKSProxy( proxy_url="socks5://localhost:8080/", network_backend=network_backend, ) as proxy: # Sending a request, which the proxy rejects with pytest.raises(ProxyError) as exc_info: proxy.request("GET", "https://example.com/") assert (str(exc_info.value) == "Proxy Server could not connect: Connection refused.") assert not proxy.connections
def test_http_connection(): origin = Origin(b"https", b"example.com", 443) network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with HTTPConnection(origin=origin, network_backend=network_backend, keepalive_expiry=5.0) as conn: assert not conn.is_idle() assert not conn.is_closed() assert not conn.is_available() assert not conn.has_expired() assert repr(conn) == "<HTTPConnection [CONNECTING]>" with conn.stream("GET", "https://example.com/") as response: assert ( repr(conn) == "<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ) response.read() assert response.status == 200 assert response.content == b"Hello, world!" assert conn.is_idle() assert not conn.is_closed() assert conn.is_available() assert not conn.has_expired() assert ( repr(conn) == "<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>" )
def test_connection_pool_with_http_exception(): """ HTTP/1.1 requests that result in an exception during the connection should not be returned to the connection pool. """ network_backend = MockBackend([b"Wait, this isn't valid HTTP!"]) called = [] def trace(name, kwargs): called.append(name) with ConnectionPool(network_backend=network_backend) as pool: # Sending an initial request, which once complete will not return to the pool. with pytest.raises(Exception): pool.request("GET", "https://example.com/", extensions={"trace": trace}) info = [repr(c) for c in pool.connections] assert info == [] assert called == [ "connection.connect_tcp.started", "connection.connect_tcp.complete", "connection.start_tls.started", "connection.start_tls.complete", "http11.send_request_headers.started", "http11.send_request_headers.complete", "http11.send_request_body.started", "http11.send_request_body.complete", "http11.receive_response_headers.started", "http11.receive_response_headers.failed", "http11.response_closed.started", "http11.response_closed.complete", ]
def test_connection_pool_with_keepalive(): """ By default HTTP/1.1 requests should be returned to the connection pool. """ network_backend = MockBackend([ b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", b"HTTP/1.1 200 OK\r\n", b"Content-Type: plain/text\r\n", b"Content-Length: 13\r\n", b"\r\n", b"Hello, world!", ]) with ConnectionPool(network_backend=network_backend, ) as pool: # Sending an intial request, which once complete will return to the pool, IDLE. with pool.stream("GET", "https://example.com/") as response: info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 1]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 1]>" ] # Sending a second request to the same origin will reuse the existing IDLE connection. with pool.stream("GET", "https://example.com/") as response: info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, ACTIVE, Request Count: 2]>" ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 2]>" ] # Sending a request to a different origin will not reuse the existing IDLE connection. with pool.stream("GET", "http://example.com/") as response: info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['http://example.com:80', HTTP/1.1, ACTIVE, Request Count: 1]>", "<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 2]>", ] response.read() assert response.status == 200 assert response.content == b"Hello, world!" info = [repr(c) for c in pool.connections] assert info == [ "<HTTPConnection ['http://example.com:80', HTTP/1.1, IDLE, Request Count: 1]>", "<HTTPConnection ['https://example.com:443', HTTP/1.1, IDLE, Request Count: 2]>", ]