def test_broken_socket_detection_many_open_files(backend: str,
server: Server) -> None:
"""
Regression test for: https://github.com/encode/httpcore/issues/182
"""
with httpcore.SyncConnectionPool(backend=backend) as http:
# * First attempt will be successful because it will grab the last
# available fd before what select() supports on the platform.
# * Second attempt would have failed without a fix, due to a "filedescriptor
# out of range in select()" exception.
for _ in range(2):
(
status_code,
response_headers,
stream,
extensions,
) = http.handle_request(
method=b"GET",
url=(b"http", *server.netloc, b"/"),
headers=[server.host_header],
stream=httpcore.ByteStream(b""),
extensions={},
)
read_body(stream)
assert status_code == 200
reason_phrase = b"OK" if server.sends_reason else b""
assert extensions == {
"http_version": b"HTTP/1.1",
"reason_phrase": reason_phrase,
}
origin = (b"http", *server.netloc)
assert len(http._connections[origin]) == 1 # type: ignore
def test_http_request_cannot_reuse_dropped_connection() -> None:
with httpcore.SyncConnectionPool() as http:
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
http_version, status_code, reason, headers, stream = http.request(
method, url, headers)
body = read_body(stream)
assert http_version == b"HTTP/1.1"
assert status_code == 200
assert reason == b"OK"
assert len(http._connections[url[:3]]) == 1 # type: ignore
# Mock the connection as having been dropped.
connection = list(http._connections[url[:3]])[0] # type: ignore
connection.is_connection_dropped = lambda: True
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
http_version, status_code, reason, headers, stream = http.request(
method, url, headers)
body = read_body(stream)
assert http_version == b"HTTP/1.1"
assert status_code == 200
assert reason == b"OK"
assert len(http._connections[url[:3]]) == 1 # type: ignore
def init_dispatch(
self,
verify: VerifyTypes = True,
cert: CertTypes = None,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
dispatch: httpcore.SyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
) -> httpcore.SyncHTTPTransport:
if dispatch is not None:
return dispatch
if app is not None:
return WSGIDispatch(app=app)
ssl_context = SSLConfig(verify=verify, cert=cert,
trust_env=trust_env).ssl_context
max_keepalive = pool_limits.soft_limit
max_connections = pool_limits.hard_limit
return httpcore.SyncConnectionPool(
ssl_context=ssl_context,
max_keepalive=max_keepalive,
max_connections=max_connections,
)
async def test_httpcore_request():
async with MockTransport() as transport:
transport.add("GET", "https://foo.bar/", content="foobar")
with httpcore.SyncConnectionPool() as http:
(http_version, status_code, reason_phrase, headers, stream,) = http.request(
method=b"GET", url=(b"https", b"foo.bar", 443, b"/"),
)
body = b"".join([chunk for chunk in stream])
stream.close()
assert body == b"foobar"
async with httpcore.AsyncConnectionPool() as http:
(
http_version,
status_code,
reason_phrase,
headers,
stream,
) = await http.request(
method=b"GET", url=(b"https", b"foo.bar", 443, b"/"),
)
body = b"".join([chunk async for chunk in stream])
await stream.aclose()
assert body == b"foobar"
def test_threadsafe_basic(server: Server, http2: bool) -> None:
"""
The sync connection pool can be used to perform requests concurrently using
threads.
Also a regression test for: https://github.com/encode/httpx/issues/1393
"""
with httpcore.SyncConnectionPool(http2=http2) as http:
def request(http: httpcore.SyncHTTPTransport) -> int:
status_code, headers, stream, extensions = http.handle_request(
method=b"GET",
url=(b"http", *server.netloc, b"/"),
headers=[server.host_header],
stream=httpcore.ByteStream(b""),
extensions={},
)
read_body(stream)
return status_code
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
futures = [executor.submit(request, http) for _ in range(10)]
num_results = 0
for future in concurrent.futures.as_completed(futures):
status_code = future.result()
assert status_code == 200
num_results += 1
assert num_results == 10
def _init_transport(
self,
verify: VerifyTypes = True,
cert: CertTypes = None,
http2: bool = False,
limits: Limits = DEFAULT_LIMITS,
transport: httpcore.SyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
) -> httpcore.SyncHTTPTransport:
if transport is not None:
return transport
if app is not None:
return WSGITransport(app=app)
ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env)
return httpcore.SyncConnectionPool(
ssl_context=ssl_context,
max_connections=limits.max_connections,
max_keepalive_connections=limits.max_keepalive_connections,
keepalive_expiry=KEEPALIVE_EXPIRY,
http2=http2,
)
def test_max_keepalive_connections_handled_correctly(max_keepalive: int,
connections_number: int,
backend: str,
server: Server) -> None:
with httpcore.SyncConnectionPool(max_keepalive_connections=max_keepalive,
keepalive_expiry=60,
backend=backend) as http:
method = b"GET"
url = (b"http", *server.netloc, b"/")
headers = [server.host_header]
connections_streams = []
for _ in range(connections_number):
_, _, stream, _ = http.request(method, url, headers)
connections_streams.append(stream)
try:
for i in range(len(connections_streams)):
read_body(connections_streams[i])
finally:
stats = http.get_connection_info()
connections_in_pool = next(iter(stats.values()))
assert len(connections_in_pool) == min(connections_number,
max_keepalive)
def test_request_unsupported_protocol() -> None:
with httpcore.SyncConnectionPool() as http:
method = b"GET"
url = (b"ftp", b"example.org", 443, b"/")
headers = [(b"host", b"example.org")]
with pytest.raises(httpcore.UnsupportedProtocol):
http.request(method, url, headers)
def test_connection_pool_get_connection_info(
http2: bool,
keepalive_expiry: float,
expected_during_active: dict,
expected_during_idle: dict,
) -> None:
with httpcore.SyncConnectionPool(
http2=http2, keepalive_expiry=keepalive_expiry) as http:
method = b"GET"
url = (b"https", b"example.org", 443, b"/")
headers = [(b"host", b"example.org")]
_, _, _, _, stream_1 = http.request(method, url, headers)
_, _, _, _, stream_2 = http.request(method, url, headers)
try:
stats = http.get_connection_info()
assert stats == expected_during_active
finally:
read_body(stream_1)
read_body(stream_2)
stats = http.get_connection_info()
assert stats == expected_during_idle
stats = http.get_connection_info()
assert stats == {}
def test_http_request_cannot_reuse_dropped_connection(backend: str,
server: Server) -> None:
with httpcore.SyncConnectionPool(backend=backend) as http:
method = b"GET"
url = (b"http", *server.netloc, b"/")
headers = [server.host_header]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
reason = "OK" if server.sends_reason else ""
assert ext == {"http_version": "HTTP/1.1", "reason": reason}
assert len(http._connections[url[:3]]) == 1 # type: ignore
# Mock the connection as having been dropped.
connection = list(http._connections[url[:3]])[0] # type: ignore
connection.is_socket_readable = lambda: True # type: ignore
method = b"GET"
url = (b"http", *server.netloc, b"/")
headers = [server.host_header]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
reason = "OK" if server.sends_reason else ""
assert ext == {"http_version": "HTTP/1.1", "reason": reason}
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_connection_pool_get_connection_info(
http2: bool,
keepalive_expiry: float,
expected_during_active: dict,
expected_during_idle: dict,
backend: str,
https_server: Server,
) -> None:
with httpcore.SyncConnectionPool(http2=http2,
keepalive_expiry=keepalive_expiry,
backend=backend) as http:
method = b"GET"
url = (b"https", *https_server.netloc, b"/")
headers = [https_server.host_header]
_, _, stream_1, _ = http.request(method, url, headers)
_, _, stream_2, _ = http.request(method, url, headers)
try:
stats = http.get_connection_info()
assert stats == expected_during_active
finally:
read_body(stream_1)
read_body(stream_2)
stats = http.get_connection_info()
assert stats == expected_during_idle
stats = http.get_connection_info()
assert stats == {}
def init_transport(
self,
verify: VerifyTypes = True,
cert: CertTypes = None,
http2: bool = False,
pool_limits: PoolLimits = DEFAULT_POOL_LIMITS,
transport: httpcore.SyncHTTPTransport = None,
app: typing.Callable = None,
trust_env: bool = True,
) -> httpcore.SyncHTTPTransport:
if transport is not None:
return transport
if app is not None:
return WSGITransport(app=app)
ssl_context = SSLConfig(verify=verify, cert=cert,
trust_env=trust_env).ssl_context
return httpcore.SyncConnectionPool(
ssl_context=ssl_context,
max_keepalive=pool_limits.max_keepalive,
max_connections=pool_limits.max_connections,
http2=http2,
)
def test_retries_exceeded(server: Server) -> None:
"""
When retries are enabled and connecting failures more than the configured number
of retries, connect exceptions are raised.
"""
backend = SyncMockBackend()
retries = 1
with httpcore.SyncConnectionPool(
retries=retries, max_keepalive_connections=0, backend=backend
) as http:
response = http.handle_request(
method=b"GET",
url=(b"http", *server.netloc, b"/"),
headers=[server.host_header],
stream=httpcore.ByteStream(b""),
extensions={},
)
status_code, _, stream, _ = response
assert status_code == 200
read_body(stream)
# First failure is retried on, second one isn't.
backend.push(httpcore.ConnectError(), httpcore.ConnectTimeout())
with pytest.raises(httpcore.ConnectTimeout):
http.handle_request(
method=b"GET",
url=(b"http", *server.netloc, b"/"),
headers=[server.host_header],
stream=httpcore.ByteStream(b""),
extensions={},
)
def test_http_request_cannot_reuse_dropped_connection(backend: str) -> None:
with httpcore.SyncConnectionPool(backend=backend) as http:
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
assert ext == {"http_version": "HTTP/1.1", "reason": "OK"}
assert len(http._connections[url[:3]]) == 1 # type: ignore
# Mock the connection as having been dropped.
connection = list(http._connections[url[:3]])[0] # type: ignore
connection.is_connection_dropped = lambda: True # type: ignore
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
assert ext == {"http_version": "HTTP/1.1", "reason": "OK"}
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_http_request_reuse_connection() -> None:
with httpcore.SyncConnectionPool() as http:
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
http_version, status_code, reason, headers, stream = http.request(
method, url, headers)
body = read_body(stream)
assert http_version == b"HTTP/1.1"
assert status_code == 200
assert reason == b"OK"
assert len(http._connections[url[:3]]) == 1 # type: ignore
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
http_version, status_code, reason, headers, stream = http.request(
method, url, headers)
body = read_body(stream)
assert http_version == b"HTTP/1.1"
assert status_code == 200
assert reason == b"OK"
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_cannot_connect_tcp(backend: str, url) -> None:
"""
A properly wrapped error is raised when connecting to the server fails.
"""
with httpcore.SyncConnectionPool(backend=backend) as http:
method = b"GET"
with pytest.raises(httpcore.ConnectError):
http.request(method, url)
def test_connection_pool_get_connection_info(http2, expected) -> None:
with httpcore.SyncConnectionPool(http2=http2) as http:
method = b"GET"
url = (b"https", b"example.org", 443, b"/")
headers = [(b"host", b"example.org")]
for _ in range(2):
_ = http.request(method, url, headers)
stats = http.get_connection_info()
assert stats == {"https://example.org": expected}
def test_cannot_connect_uds(backend: str) -> None:
"""
A properly wrapped error is raised when connecting to the UDS server fails.
"""
uds = "/tmp/doesnotexist.sock"
method = b"GET"
url = (b"http", b"localhost", None, b"/")
with httpcore.SyncConnectionPool(backend=backend, uds=uds) as http:
with pytest.raises(httpcore.ConnectError):
http.request(method, url)
def test_request_unsupported_protocol(backend: str) -> None:
with httpcore.SyncConnectionPool(backend=backend) as http:
with pytest.raises(httpcore.UnsupportedProtocol):
http.handle_request(
method=b"GET",
url=(b"ftp", b"example.org", 443, b"/"),
headers=[(b"host", b"example.org")],
stream=httpcore.ByteStream(b""),
extensions={},
)
def test_closing_http_request(backend: str) -> None:
with httpcore.SyncConnectionPool(backend=backend) as http:
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org"), (b"connection", b"close")]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
assert ext == {"http_version": "HTTP/1.1", "reason": "OK"}
assert url[:3] not in http._connections # type: ignore
def test_explicit_backend_name() -> None:
with httpcore.SyncConnectionPool(backend=lookup_sync_backend()) as http:
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org")]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
assert ext == {"http_version": "HTTP/1.1", "reason": "OK"}
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_http2_request(backend: str) -> None:
with httpcore.SyncConnectionPool(backend=backend, http2=True) as http:
method = b"GET"
url = (b"https", b"example.org", 443, b"/")
headers = [(b"host", b"example.org")]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
assert ext == {"http_version": "HTTP/2"}
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_https_request(backend: str, https_server: Server) -> None:
with httpcore.SyncConnectionPool(backend=backend) as http:
method = b"GET"
url = (b"https", *https_server.netloc, b"/")
headers = [https_server.host_header]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
assert ext == {"http_version": "HTTP/1.1", "reason": "OK"}
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_retries_enabled(server: Server) -> None:
"""
When retries are enabled, connection failures are retried on with
a fixed exponential backoff.
"""
method = b"GET"
url = (b"http", *server.netloc, b"/")
headers = [server.host_header]
backend = SyncMockBackend()
retries = 10 # Large enough to not run out of retries within this test.
with httpcore.SyncConnectionPool(retries=retries,
max_keepalive_connections=0,
backend=backend) as http:
# Standard case, no failures.
response = http.request(method, url, headers)
assert backend.pop_open_tcp_stream_intervals() == []
status_code, _, stream, _ = response
assert status_code == 200
read_body(stream)
# One failure, then success.
backend.push(httpcore.ConnectError(), None)
response = http.request(method, url, headers)
assert backend.pop_open_tcp_stream_intervals() == [
pytest.approx(0, abs=1e-3), # Retry immediately.
]
status_code, _, stream, _ = response
assert status_code == 200
read_body(stream)
# Three failures, then success.
backend.push(
httpcore.ConnectError(),
httpcore.ConnectTimeout(),
httpcore.ConnectTimeout(),
None,
)
response = http.request(method, url, headers)
assert backend.pop_open_tcp_stream_intervals() == [
pytest.approx(0, abs=1e-3), # Retry immediately.
pytest.approx(0.5, rel=0.1), # First backoff.
pytest.approx(1.0, rel=0.1), # Second (increased) backoff.
]
status_code, _, stream, _ = response
assert status_code == 200
read_body(stream)
# Non-connect exceptions are not retried on.
backend.push(httpcore.ReadTimeout(), httpcore.NetworkError())
with pytest.raises(httpcore.ReadTimeout):
http.request(method, url, headers)
with pytest.raises(httpcore.NetworkError):
http.request(method, url, headers)
def test_connection_timeout_tcp(backend: str, server: Server) -> None:
# we try to access http server using https. It caused some SSL timeouts
# in TLSStream.wrap inside inside AnyIOBackend.open_tcp_stream
method = b"GET"
url = (b"https", *server.netloc, b"/")
headers = [server.host_header]
ext = {"timeout": {"connect": 0.1}}
with httpcore.SyncConnectionPool(backend=backend) as http:
with pytest.raises(httpcore.ConnectTimeout):
http.request(method, url, headers, ext=ext)
def test_explicit_backend_name(server: Server) -> None:
with httpcore.SyncConnectionPool(backend=lookup_sync_backend()) as http:
method = b"GET"
url = (b"http", *server.netloc, b"/")
headers = [server.host_header]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
reason = "OK" if server.sends_reason else ""
assert ext == {"http_version": "HTTP/1.1", "reason": reason}
assert len(http._connections[url[:3]]) == 1 # type: ignore
def test_closing_http_request(backend: str, server: Server) -> None:
with httpcore.SyncConnectionPool(backend=backend) as http:
method = b"GET"
url = (b"http", *server.netloc, b"/")
headers = [server.host_header, (b"connection", b"close")]
status_code, headers, stream, ext = http.request(method, url, headers)
read_body(stream)
assert status_code == 200
reason = "OK" if server.sends_reason else ""
assert ext == {"http_version": "HTTP/1.1", "reason": reason}
assert url[:3] not in http._connections # type: ignore
def test_connection_timeout_uds(backend: str, uds_server: Server,
uds: str) -> None:
# we try to access http server using https. It caused some SSL timeouts
# in TLSStream.wrap inside AnyIOBackend.open_uds_stream
method = b"GET"
url = (b"https", b"localhost", None, b"/")
headers = [(b"host", b"localhost")]
ext = {"timeout": {"connect": 0.1}}
with httpcore.SyncConnectionPool(uds=uds, backend=backend) as http:
with pytest.raises(httpcore.ConnectTimeout):
http.request(method, url, headers, ext=ext)
def test_closing_http_request() -> None:
with httpcore.SyncConnectionPool() as http:
method = b"GET"
url = (b"http", b"example.org", 80, b"/")
headers = [(b"host", b"example.org"), (b"connection", b"close")]
http_version, status_code, reason, headers, stream = http.request(
method, url, headers)
read_body(stream)
assert http_version == b"HTTP/1.1"
assert status_code == 200
assert reason == b"OK"
assert url[:3] not in http._connections # type: ignore
def test_http_request_unix_domain_socket(uds_server: Server,
backend: str) -> None:
uds = uds_server.config.uds
assert uds is not None
with httpcore.SyncConnectionPool(uds=uds, backend=backend) as http:
method = b"GET"
url = (b"http", b"localhost", None, b"/")
headers = [(b"host", b"localhost")]
status_code, headers, stream, ext = http.request(method, url, headers)
assert status_code == 200
assert ext == {"http_version": "HTTP/1.1", "reason": "OK"}
body = read_body(stream)
assert body == b"Hello, world!"