def test_broken_socket_detection_many_open_files(backend: str, server: Server) -> None: """ Regression test for: https://github.com/encode/httpcore/issues/182 """ with httpcore.SyncConnectionPool(backend=backend) as http: # * First attempt will be successful because it will grab the last # available fd before what select() supports on the platform. # * Second attempt would have failed without a fix, due to a "filedescriptor # out of range in select()" exception. for _ in range(2): ( status_code, response_headers, stream, extensions, ) = http.handle_request( method=b"GET", url=(b"http", *server.netloc, b"/"), headers=[server.host_header], stream=httpcore.ByteStream(b""), extensions={}, ) read_body(stream) assert status_code == 200 reason_phrase = b"OK" if server.sends_reason else b"" assert extensions == { "http_version": b"HTTP/1.1", "reason_phrase": reason_phrase, } origin = (b"http", *server.netloc) assert len(http._connections[origin]) == 1 # type: ignore
def test_http_request_cannot_reuse_dropped_connection() -> None: with httpcore.SyncConnectionPool() as http: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] http_version, status_code, reason, headers, stream = http.request( method, url, headers) body = read_body(stream) assert http_version == b"HTTP/1.1" assert status_code == 200 assert reason == b"OK" assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. connection = list(http._connections[url[:3]])[0] # type: ignore connection.is_connection_dropped = lambda: True method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] http_version, status_code, reason, headers, stream = http.request( method, url, headers) body = read_body(stream) assert http_version == b"HTTP/1.1" assert status_code == 200 assert reason == b"OK" assert len(http._connections[url[:3]]) == 1 # type: ignore
def init_dispatch( self, verify: VerifyTypes = True, cert: CertTypes = None, pool_limits: PoolLimits = DEFAULT_POOL_LIMITS, dispatch: httpcore.SyncHTTPTransport = None, app: typing.Callable = None, trust_env: bool = True, ) -> httpcore.SyncHTTPTransport: if dispatch is not None: return dispatch if app is not None: return WSGIDispatch(app=app) ssl_context = SSLConfig(verify=verify, cert=cert, trust_env=trust_env).ssl_context max_keepalive = pool_limits.soft_limit max_connections = pool_limits.hard_limit return httpcore.SyncConnectionPool( ssl_context=ssl_context, max_keepalive=max_keepalive, max_connections=max_connections, )
async def test_httpcore_request(): async with MockTransport() as transport: transport.add("GET", "https://foo.bar/", content="foobar") with httpcore.SyncConnectionPool() as http: (http_version, status_code, reason_phrase, headers, stream,) = http.request( method=b"GET", url=(b"https", b"foo.bar", 443, b"/"), ) body = b"".join([chunk for chunk in stream]) stream.close() assert body == b"foobar" async with httpcore.AsyncConnectionPool() as http: ( http_version, status_code, reason_phrase, headers, stream, ) = await http.request( method=b"GET", url=(b"https", b"foo.bar", 443, b"/"), ) body = b"".join([chunk async for chunk in stream]) await stream.aclose() assert body == b"foobar"
def test_threadsafe_basic(server: Server, http2: bool) -> None: """ The sync connection pool can be used to perform requests concurrently using threads. Also a regression test for: https://github.com/encode/httpx/issues/1393 """ with httpcore.SyncConnectionPool(http2=http2) as http: def request(http: httpcore.SyncHTTPTransport) -> int: status_code, headers, stream, extensions = http.handle_request( method=b"GET", url=(b"http", *server.netloc, b"/"), headers=[server.host_header], stream=httpcore.ByteStream(b""), extensions={}, ) read_body(stream) return status_code with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: futures = [executor.submit(request, http) for _ in range(10)] num_results = 0 for future in concurrent.futures.as_completed(futures): status_code = future.result() assert status_code == 200 num_results += 1 assert num_results == 10
def _init_transport( self, verify: VerifyTypes = True, cert: CertTypes = None, http2: bool = False, limits: Limits = DEFAULT_LIMITS, transport: httpcore.SyncHTTPTransport = None, app: typing.Callable = None, trust_env: bool = True, ) -> httpcore.SyncHTTPTransport: if transport is not None: return transport if app is not None: return WSGITransport(app=app) ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) return httpcore.SyncConnectionPool( ssl_context=ssl_context, max_connections=limits.max_connections, max_keepalive_connections=limits.max_keepalive_connections, keepalive_expiry=KEEPALIVE_EXPIRY, http2=http2, )
def test_max_keepalive_connections_handled_correctly(max_keepalive: int, connections_number: int, backend: str, server: Server) -> None: with httpcore.SyncConnectionPool(max_keepalive_connections=max_keepalive, keepalive_expiry=60, backend=backend) as http: method = b"GET" url = (b"http", *server.netloc, b"/") headers = [server.host_header] connections_streams = [] for _ in range(connections_number): _, _, stream, _ = http.request(method, url, headers) connections_streams.append(stream) try: for i in range(len(connections_streams)): read_body(connections_streams[i]) finally: stats = http.get_connection_info() connections_in_pool = next(iter(stats.values())) assert len(connections_in_pool) == min(connections_number, max_keepalive)
def test_request_unsupported_protocol() -> None: with httpcore.SyncConnectionPool() as http: method = b"GET" url = (b"ftp", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] with pytest.raises(httpcore.UnsupportedProtocol): http.request(method, url, headers)
def test_connection_pool_get_connection_info( http2: bool, keepalive_expiry: float, expected_during_active: dict, expected_during_idle: dict, ) -> None: with httpcore.SyncConnectionPool( http2=http2, keepalive_expiry=keepalive_expiry) as http: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] _, _, _, _, stream_1 = http.request(method, url, headers) _, _, _, _, stream_2 = http.request(method, url, headers) try: stats = http.get_connection_info() assert stats == expected_during_active finally: read_body(stream_1) read_body(stream_2) stats = http.get_connection_info() assert stats == expected_during_idle stats = http.get_connection_info() assert stats == {}
def test_http_request_cannot_reuse_dropped_connection(backend: str, server: Server) -> None: with httpcore.SyncConnectionPool(backend=backend) as http: method = b"GET" url = (b"http", *server.netloc, b"/") headers = [server.host_header] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 reason = "OK" if server.sends_reason else "" assert ext == {"http_version": "HTTP/1.1", "reason": reason} assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. connection = list(http._connections[url[:3]])[0] # type: ignore connection.is_socket_readable = lambda: True # type: ignore method = b"GET" url = (b"http", *server.netloc, b"/") headers = [server.host_header] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 reason = "OK" if server.sends_reason else "" assert ext == {"http_version": "HTTP/1.1", "reason": reason} assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_connection_pool_get_connection_info( http2: bool, keepalive_expiry: float, expected_during_active: dict, expected_during_idle: dict, backend: str, https_server: Server, ) -> None: with httpcore.SyncConnectionPool(http2=http2, keepalive_expiry=keepalive_expiry, backend=backend) as http: method = b"GET" url = (b"https", *https_server.netloc, b"/") headers = [https_server.host_header] _, _, stream_1, _ = http.request(method, url, headers) _, _, stream_2, _ = http.request(method, url, headers) try: stats = http.get_connection_info() assert stats == expected_during_active finally: read_body(stream_1) read_body(stream_2) stats = http.get_connection_info() assert stats == expected_during_idle stats = http.get_connection_info() assert stats == {}
def init_transport( self, verify: VerifyTypes = True, cert: CertTypes = None, http2: bool = False, pool_limits: PoolLimits = DEFAULT_POOL_LIMITS, transport: httpcore.SyncHTTPTransport = None, app: typing.Callable = None, trust_env: bool = True, ) -> httpcore.SyncHTTPTransport: if transport is not None: return transport if app is not None: return WSGITransport(app=app) ssl_context = SSLConfig(verify=verify, cert=cert, trust_env=trust_env).ssl_context return httpcore.SyncConnectionPool( ssl_context=ssl_context, max_keepalive=pool_limits.max_keepalive, max_connections=pool_limits.max_connections, http2=http2, )
def test_retries_exceeded(server: Server) -> None: """ When retries are enabled and connecting failures more than the configured number of retries, connect exceptions are raised. """ backend = SyncMockBackend() retries = 1 with httpcore.SyncConnectionPool( retries=retries, max_keepalive_connections=0, backend=backend ) as http: response = http.handle_request( method=b"GET", url=(b"http", *server.netloc, b"/"), headers=[server.host_header], stream=httpcore.ByteStream(b""), extensions={}, ) status_code, _, stream, _ = response assert status_code == 200 read_body(stream) # First failure is retried on, second one isn't. backend.push(httpcore.ConnectError(), httpcore.ConnectTimeout()) with pytest.raises(httpcore.ConnectTimeout): http.handle_request( method=b"GET", url=(b"http", *server.netloc, b"/"), headers=[server.host_header], stream=httpcore.ByteStream(b""), extensions={}, )
def test_http_request_cannot_reuse_dropped_connection(backend: str) -> None: with httpcore.SyncConnectionPool(backend=backend) as http: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore # Mock the connection as having been dropped. connection = list(http._connections[url[:3]])[0] # type: ignore connection.is_connection_dropped = lambda: True # type: ignore method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_http_request_reuse_connection() -> None: with httpcore.SyncConnectionPool() as http: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] http_version, status_code, reason, headers, stream = http.request( method, url, headers) body = read_body(stream) assert http_version == b"HTTP/1.1" assert status_code == 200 assert reason == b"OK" assert len(http._connections[url[:3]]) == 1 # type: ignore method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] http_version, status_code, reason, headers, stream = http.request( method, url, headers) body = read_body(stream) assert http_version == b"HTTP/1.1" assert status_code == 200 assert reason == b"OK" assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_cannot_connect_tcp(backend: str, url) -> None: """ A properly wrapped error is raised when connecting to the server fails. """ with httpcore.SyncConnectionPool(backend=backend) as http: method = b"GET" with pytest.raises(httpcore.ConnectError): http.request(method, url)
def test_connection_pool_get_connection_info(http2, expected) -> None: with httpcore.SyncConnectionPool(http2=http2) as http: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] for _ in range(2): _ = http.request(method, url, headers) stats = http.get_connection_info() assert stats == {"https://example.org": expected}
def test_cannot_connect_uds(backend: str) -> None: """ A properly wrapped error is raised when connecting to the UDS server fails. """ uds = "/tmp/doesnotexist.sock" method = b"GET" url = (b"http", b"localhost", None, b"/") with httpcore.SyncConnectionPool(backend=backend, uds=uds) as http: with pytest.raises(httpcore.ConnectError): http.request(method, url)
def test_request_unsupported_protocol(backend: str) -> None: with httpcore.SyncConnectionPool(backend=backend) as http: with pytest.raises(httpcore.UnsupportedProtocol): http.handle_request( method=b"GET", url=(b"ftp", b"example.org", 443, b"/"), headers=[(b"host", b"example.org")], stream=httpcore.ByteStream(b""), extensions={}, )
def test_closing_http_request(backend: str) -> None: with httpcore.SyncConnectionPool(backend=backend) as http: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org"), (b"connection", b"close")] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert url[:3] not in http._connections # type: ignore
def test_explicit_backend_name() -> None: with httpcore.SyncConnectionPool(backend=lookup_sync_backend()) as http: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org")] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_http2_request(backend: str) -> None: with httpcore.SyncConnectionPool(backend=backend, http2=True) as http: method = b"GET" url = (b"https", b"example.org", 443, b"/") headers = [(b"host", b"example.org")] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 assert ext == {"http_version": "HTTP/2"} assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_https_request(backend: str, https_server: Server) -> None: with httpcore.SyncConnectionPool(backend=backend) as http: method = b"GET" url = (b"https", *https_server.netloc, b"/") headers = [https_server.host_header] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_retries_enabled(server: Server) -> None: """ When retries are enabled, connection failures are retried on with a fixed exponential backoff. """ method = b"GET" url = (b"http", *server.netloc, b"/") headers = [server.host_header] backend = SyncMockBackend() retries = 10 # Large enough to not run out of retries within this test. with httpcore.SyncConnectionPool(retries=retries, max_keepalive_connections=0, backend=backend) as http: # Standard case, no failures. response = http.request(method, url, headers) assert backend.pop_open_tcp_stream_intervals() == [] status_code, _, stream, _ = response assert status_code == 200 read_body(stream) # One failure, then success. backend.push(httpcore.ConnectError(), None) response = http.request(method, url, headers) assert backend.pop_open_tcp_stream_intervals() == [ pytest.approx(0, abs=1e-3), # Retry immediately. ] status_code, _, stream, _ = response assert status_code == 200 read_body(stream) # Three failures, then success. backend.push( httpcore.ConnectError(), httpcore.ConnectTimeout(), httpcore.ConnectTimeout(), None, ) response = http.request(method, url, headers) assert backend.pop_open_tcp_stream_intervals() == [ pytest.approx(0, abs=1e-3), # Retry immediately. pytest.approx(0.5, rel=0.1), # First backoff. pytest.approx(1.0, rel=0.1), # Second (increased) backoff. ] status_code, _, stream, _ = response assert status_code == 200 read_body(stream) # Non-connect exceptions are not retried on. backend.push(httpcore.ReadTimeout(), httpcore.NetworkError()) with pytest.raises(httpcore.ReadTimeout): http.request(method, url, headers) with pytest.raises(httpcore.NetworkError): http.request(method, url, headers)
def test_connection_timeout_tcp(backend: str, server: Server) -> None: # we try to access http server using https. It caused some SSL timeouts # in TLSStream.wrap inside inside AnyIOBackend.open_tcp_stream method = b"GET" url = (b"https", *server.netloc, b"/") headers = [server.host_header] ext = {"timeout": {"connect": 0.1}} with httpcore.SyncConnectionPool(backend=backend) as http: with pytest.raises(httpcore.ConnectTimeout): http.request(method, url, headers, ext=ext)
def test_explicit_backend_name(server: Server) -> None: with httpcore.SyncConnectionPool(backend=lookup_sync_backend()) as http: method = b"GET" url = (b"http", *server.netloc, b"/") headers = [server.host_header] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 reason = "OK" if server.sends_reason else "" assert ext == {"http_version": "HTTP/1.1", "reason": reason} assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_closing_http_request(backend: str, server: Server) -> None: with httpcore.SyncConnectionPool(backend=backend) as http: method = b"GET" url = (b"http", *server.netloc, b"/") headers = [server.host_header, (b"connection", b"close")] status_code, headers, stream, ext = http.request(method, url, headers) read_body(stream) assert status_code == 200 reason = "OK" if server.sends_reason else "" assert ext == {"http_version": "HTTP/1.1", "reason": reason} assert url[:3] not in http._connections # type: ignore
def test_connection_timeout_uds(backend: str, uds_server: Server, uds: str) -> None: # we try to access http server using https. It caused some SSL timeouts # in TLSStream.wrap inside AnyIOBackend.open_uds_stream method = b"GET" url = (b"https", b"localhost", None, b"/") headers = [(b"host", b"localhost")] ext = {"timeout": {"connect": 0.1}} with httpcore.SyncConnectionPool(uds=uds, backend=backend) as http: with pytest.raises(httpcore.ConnectTimeout): http.request(method, url, headers, ext=ext)
def test_closing_http_request() -> None: with httpcore.SyncConnectionPool() as http: method = b"GET" url = (b"http", b"example.org", 80, b"/") headers = [(b"host", b"example.org"), (b"connection", b"close")] http_version, status_code, reason, headers, stream = http.request( method, url, headers) read_body(stream) assert http_version == b"HTTP/1.1" assert status_code == 200 assert reason == b"OK" assert url[:3] not in http._connections # type: ignore
def test_http_request_unix_domain_socket(uds_server: Server, backend: str) -> None: uds = uds_server.config.uds assert uds is not None with httpcore.SyncConnectionPool(uds=uds, backend=backend) as http: method = b"GET" url = (b"http", b"localhost", None, b"/") headers = [(b"host", b"localhost")] status_code, headers, stream, ext = http.request(method, url, headers) assert status_code == 200 assert ext == {"http_version": "HTTP/1.1", "reason": "OK"} body = read_body(stream) assert body == b"Hello, world!"