def test_total_timeout(self):
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=2)
wait_for_socket(ready_event)
# This will get the socket to raise an EAGAIN on the read
timeout = Timeout(connect=3, read=SHORT_TIMEOUT)
with HTTPConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False) as pool:
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/")
block_event.set()
wait_for_socket(ready_event)
block_event.clear()
# The connect should succeed and this should hit the read timeout
timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT)
with HTTPConnectionPool(self.host,
self.port,
timeout=timeout,
retries=False) as pool:
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/")
def test_connect_timeout(self):
url = "/"
host, port = TARPIT_HOST, 80
timeout = Timeout(connect=SHORT_TIMEOUT)
# Pool-global timeout
with HTTPConnectionPool(host, port, timeout=timeout) as pool:
conn = pool._get_conn()
with pytest.raises(ConnectTimeoutError):
pool._make_request(conn, "GET", url)
# Retries
retries = Retry(connect=0)
with pytest.raises(MaxRetryError):
pool.request("GET", url, retries=retries)
# Request-specific connection timeouts
big_timeout = Timeout(read=LONG_TIMEOUT, connect=LONG_TIMEOUT)
with HTTPConnectionPool(host, port, timeout=big_timeout,
retries=False) as pool:
conn = pool._get_conn()
with pytest.raises(ConnectTimeoutError):
pool._make_request(conn, "GET", url, timeout=timeout)
pool._put_conn(conn)
with pytest.raises(ConnectTimeoutError):
pool.request("GET", url, timeout=timeout)
def test_timeout_success(self):
timeout = Timeout(connect=3, read=5, total=None)
with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool:
pool.request("GET", "/")
# This should not raise a "Timeout already started" error
pool.request("GET", "/")
with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool:
# This should also not raise a "Timeout already started" error
pool.request("GET", "/")
timeout = Timeout(total=None)
with HTTPConnectionPool(self.host, self.port, timeout=timeout) as pool:
pool.request("GET", "/")
def test_lazy_load_twice(self):
# This test is sad and confusing. Need to figure out what's
# going on with partial reads and socket reuse.
with HTTPConnectionPool(self.host,
self.port,
block=True,
maxsize=1,
timeout=2) as pool:
payload_size = 1024 * 2
first_chunk = 512
boundary = "foo"
req_data = {"count": "a" * payload_size}
resp_data = encode_multipart_formdata(req_data,
boundary=boundary)[0]
req2_data = {"count": "b" * payload_size}
resp2_data = encode_multipart_formdata(req2_data,
boundary=boundary)[0]
r1 = pool.request(
"POST",
"/echo",
fields=req_data,
multipart_boundary=boundary,
preload_content=False,
)
first_data = r1.read(first_chunk)
assert len(first_data) > 0
assert first_data == resp_data[:len(first_data)]
try:
r2 = pool.request(
"POST",
"/echo",
fields=req2_data,
multipart_boundary=boundary,
preload_content=False,
pool_timeout=0.001,
)
# This branch should generally bail here, but maybe someday it will
# work? Perhaps by some sort of magic. Consider it a TODO.
second_data = r2.read(first_chunk)
assert len(second_data) > 0
assert second_data == resp2_data[:len(second_data)]
assert r1.read() == resp_data[len(first_data):]
assert r2.read() == resp2_data[len(second_data):]
assert pool.num_requests == 2
except EmptyPoolError:
assert r1.read() == resp_data[len(first_data):]
assert pool.num_requests == 1
assert pool.num_connections == 1
def test_connection_error_retries(self):
""" ECONNREFUSED error should raise a connection error, with retries """
port = find_unused_port()
with HTTPConnectionPool(self.host, port) as pool:
with pytest.raises(MaxRetryError) as e:
pool.request("GET", "/", retries=Retry(connect=3))
assert type(e.value.reason) == NewConnectionError
def test_for_double_release(self):
MAXSIZE = 5
# Check default state
with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool:
assert pool.num_connections == 0
assert pool.pool.qsize() == MAXSIZE
# Make an empty slot for testing
pool.pool.get()
assert pool.pool.qsize() == MAXSIZE - 1
# Check state after simple request
pool.urlopen("GET", "/")
assert pool.pool.qsize() == MAXSIZE - 1
# Check state without release
pool.urlopen("GET", "/", preload_content=False)
assert pool.pool.qsize() == MAXSIZE - 2
pool.urlopen("GET", "/")
assert pool.pool.qsize() == MAXSIZE - 2
# Check state after read
pool.urlopen("GET", "/").data
assert pool.pool.qsize() == MAXSIZE - 2
pool.urlopen("GET", "/")
assert pool.pool.qsize() == MAXSIZE - 2
def test_connections_arent_released(self):
MAXSIZE = 5
with HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE) as pool:
assert pool.pool.qsize() == MAXSIZE
pool.request("GET", "/", preload_content=False)
assert pool.pool.qsize() == MAXSIZE - 1
def test_preserve_chunked_on_retry(self):
self.chunked_requests = 0
def socket_handler(listener):
for _ in range(2):
sock = listener.accept()[0]
request = consume_socket(sock)
if b"transfer-encoding: chunked" in request.split(b"\r\n"):
self.chunked_requests += 1
sock.send(
b"HTTP/1.1 429 Too Many Requests\r\n"
b"Content-Type: text/plain\r\n"
b"Retry-After: 1\r\n"
b"\r\n"
)
sock.close()
self._start_server(socket_handler)
with HTTPConnectionPool(self.host, self.port) as pool:
retries = Retry(total=1)
pool.urlopen(
"GET",
"/",
body=iter([b"chunk1", b"chunk2"]),
preload_content=False,
retries=retries,
)
assert self.chunked_requests == 2
def test_preserve_chunked_on_broken_connection(self):
self.chunked_requests = 0
def socket_handler(listener):
for i in range(2):
sock = listener.accept()[0]
request = consume_socket(sock)
if b"transfer-encoding: chunked" in request.split(b"\r\n"):
self.chunked_requests += 1
if i == 0:
# Bad HTTP version will trigger a connection close
sock.send(b"HTTP/0.5 200 OK\r\n\r\n")
else:
sock.send(b"HTTP/1.1 200 OK\r\n\r\n")
sock.close()
self._start_server(socket_handler)
with HTTPConnectionPool(self.host, self.port) as pool:
retries = Retry(read=1)
pool.urlopen(
"GET",
"/",
body=iter([b"chunk1", b"chunk2"]),
preload_content=False,
retries=retries,
)
assert self.chunked_requests == 2
def test_connection_count(self):
with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool:
pool.request("GET", "/")
pool.request("GET", "/")
pool.request("GET", "/")
assert pool.num_connections == 1
assert pool.num_requests == 3
def test_total_applies_connect(self):
host, port = TARPIT_HOST, 80
timeout = Timeout(total=None, connect=SHORT_TIMEOUT)
with HTTPConnectionPool(host, port, timeout=timeout) as pool:
conn = pool._get_conn()
with pytest.raises(ConnectTimeoutError):
pool._make_request(conn, "GET", "/")
timeout = Timeout(connect=3, read=5, total=SHORT_TIMEOUT)
with HTTPConnectionPool(host, port, timeout=timeout) as pool:
try:
conn = pool._get_conn()
with pytest.raises(ConnectTimeoutError):
pool._make_request(conn, "GET", "/")
finally:
conn.close()
def test_source_address_error(self):
for addr in INVALID_SOURCE_ADDRESSES:
with HTTPConnectionPool(self.host,
self.port,
source_address=addr,
retries=False) as pool:
with pytest.raises(NewConnectionError):
pool.request("GET", "/source_address?{0}".format(addr))
def test_connection_count_bigpool(self):
with HTTPConnectionPool(self.host, self.port, maxsize=16) as http_pool:
http_pool.request("GET", "/")
http_pool.request("GET", "/")
http_pool.request("GET", "/")
assert http_pool.num_connections == 1
assert http_pool.num_requests == 3
def test_keepalive(self):
with HTTPConnectionPool(self.host, self.port, block=True,
maxsize=1) as pool:
r = pool.request("GET", "/keepalive?close=0")
r = pool.request("GET", "/keepalive?close=0")
assert r.status == 200
assert pool.num_connections == 1
assert pool.num_requests == 2
def test_provides_default_host_header(self):
self.start_chunked_handler()
chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"]
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen("GET", "/", chunks)
header_block = self.buffer.split(b"\r\n\r\n", 1)[0].lower()
header_lines = header_block.split(b"\r\n")[1:]
host_headers = [x for x in header_lines if x.startswith(b"host")]
assert len(host_headers) == 1
def test_source_address(self):
for addr, is_ipv6 in VALID_SOURCE_ADDRESSES:
if is_ipv6 and not HAS_IPV6_AND_DNS:
warnings.warn("No IPv6 support: skipping.", NoIPv6Warning)
continue
with HTTPConnectionPool(self.host,
self.port,
source_address=addr,
retries=False) as pool:
r = pool.request("GET", "/source_address")
assert r.data == b(addr[0])
def test_partial_response(self):
with HTTPConnectionPool(self.host, self.port, maxsize=1) as pool:
req_data = {"lol": "cat"}
resp_data = urlencode(req_data).encode("utf-8")
r = pool.request("GET",
"/echo",
fields=req_data,
preload_content=False)
assert r.read(5) == resp_data[:5]
assert r.read() == resp_data[5:]
def test_create_connection_timeout(self):
self.start_basic_handler(block_send=Event(),
num=0) # needed for self.port
timeout = Timeout(connect=SHORT_TIMEOUT, total=LONG_TIMEOUT)
with HTTPConnectionPool(TARPIT_HOST,
self.port,
timeout=timeout,
retries=False) as pool:
conn = pool._new_conn()
with pytest.raises(ConnectTimeoutError):
conn.connect(connect_timeout=timeout.connect_timeout)
def test_keepalive_close(self):
with HTTPConnectionPool(self.host,
self.port,
block=True,
maxsize=1,
timeout=2) as pool:
r = pool.request("GET",
"/keepalive?close=1",
retries=0,
headers={"Connection": "close"})
assert pool.num_connections == 1
# The dummyserver will have responded with Connection:close,
# and httplib will properly cleanup the socket.
# We grab the HTTPConnection object straight from the Queue,
# because _get_conn() is where the check & reset occurs
# pylint: disable-msg=W0212
conn = pool.pool.get()
assert conn._sock is None
pool._put_conn(conn)
# Now with keep-alive
r = pool.request(
"GET",
"/keepalive?close=0",
retries=0,
headers={"Connection": "keep-alive"},
)
# The dummyserver responded with Connection:keep-alive, the connection
# persists.
conn = pool.pool.get()
assert conn._sock is not None
pool._put_conn(conn)
# Another request asking the server to close the connection. This one
# should get cleaned up for the next request.
r = pool.request("GET",
"/keepalive?close=1",
retries=0,
headers={"Connection": "close"})
assert r.status == 200
conn = pool.pool.get()
assert conn._sock is None
pool._put_conn(conn)
# Next request
r = pool.request("GET", "/keepalive?close=0")
def test_nagle(self):
""" Test that connections have TCP_NODELAY turned on """
# This test needs to be here in order to be run. socket.create_connection actually tries
# to connect to the host provided so we need a dummyserver to be running.
with HTTPConnectionPool(self.host, self.port) as pool:
try:
conn = pool._get_conn()
pool._make_request(conn, "GET", "/")
tcp_nodelay_setting = conn._sock.getsockopt(
socket.IPPROTO_TCP, socket.TCP_NODELAY)
assert tcp_nodelay_setting
finally:
conn.close()
def test_disable_default_socket_options(self):
"""Test that passing None disables all socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries
# to connect to the host provided so we need a dummyserver to be running.
with HTTPConnectionPool(self.host, self.port,
socket_options=None) as pool:
conn = pool._new_conn()
conn.connect()
s = conn._sock
using_nagle = s.getsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY) == 0
assert using_nagle
s.close()
def test_chunks(self):
self.start_chunked_handler()
chunks = [b"foo", b"bar", b"", b"bazzzzzzzzzzzzzzzzzzzzzz"]
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen("GET", "/", chunks, headers=dict(DNT="1"))
assert b"transfer-encoding" in self.buffer
body = self.buffer.split(b"\r\n\r\n", 1)[1]
lines = body.split(b"\r\n")
# Empty chunks should have been skipped, as this could not be distinguished
# from terminating the transmission
for i, chunk in enumerate([c for c in chunks if c]):
assert lines[i * 2] == hex(len(chunk))[2:].encode("utf-8")
assert lines[i * 2 + 1] == chunk
def test_timeout_float(self):
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=2)
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
wait_for_socket(ready_event)
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=SHORT_TIMEOUT)
block_event.set() # Release block
# Shouldn't raise this time
wait_for_socket(ready_event)
block_event.set() # Pre-release block
pool.request("GET", "/", timeout=LONG_TIMEOUT)
def test_timeout(self):
# Requests should time out when expected
block_event = Event()
ready_event = self.start_basic_handler(block_send=block_event, num=3)
# Pool-global timeout
short_timeout = Timeout(read=SHORT_TIMEOUT)
with HTTPConnectionPool(self.host,
self.port,
timeout=short_timeout,
retries=False) as pool:
wait_for_socket(ready_event)
block_event.clear()
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/")
block_event.set() # Release request
# Request-specific timeouts should raise errors
with HTTPConnectionPool(self.host,
self.port,
timeout=short_timeout,
retries=False) as pool:
wait_for_socket(ready_event)
now = time.time()
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=LONG_TIMEOUT)
delta = time.time() - now
message = "timeout was pool-level SHORT_TIMEOUT rather than request-level LONG_TIMEOUT"
assert delta >= LONG_TIMEOUT, message
block_event.set() # Release request
# Timeout passed directly to request should raise a request timeout
wait_for_socket(ready_event)
with pytest.raises(ReadTimeoutError):
pool.request("GET", "/", timeout=SHORT_TIMEOUT)
block_event.set() # Release request
def test_socket_options(self):
"""Test that connections accept socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
with HTTPConnectionPool(
self.host,
self.port,
socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)],
) as pool:
conn = pool._new_conn()
conn.connect()
s = conn._sock
using_keepalive = s.getsockopt(socket.SOL_SOCKET,
socket.SO_KEEPALIVE) > 0
assert using_keepalive
s.close()
def _test_body(self, data):
self.start_chunked_handler()
with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
pool.urlopen("GET", "/", data)
header, body = self.buffer.split(b"\r\n\r\n", 1)
assert b"transfer-encoding: chunked" in header.split(b"\r\n")
if data:
bdata = data if isinstance(data, bytes) else data.encode("utf-8")
assert b"\r\n" + bdata + b"\r\n" in body
assert body.endswith(b"\r\n0\r\n\r\n")
len_str = body.split(b"\r\n", 1)[0]
stated_len = int(len_str, 16)
assert stated_len == len(bdata)
else:
assert body == b"0\r\n\r\n"
def test_conn_closed(self):
block_event = Event()
self.start_basic_handler(block_send=block_event, num=1)
with HTTPConnectionPool(self.host,
self.port,
timeout=SHORT_TIMEOUT,
retries=False) as pool:
conn = pool._get_conn()
pool._put_conn(conn)
try:
with pytest.raises(ReadTimeoutError):
pool.urlopen("GET", "/")
if conn._sock:
with pytest.raises(socket.error):
conn.sock.recv(1024)
finally:
pool._put_conn(conn)
block_event.set()
def test_defaults_are_applied(self):
"""Test that modifying the default socket options works."""
# This test needs to be here in order to be run. socket.create_connection actually tries
# to connect to the host provided so we need a dummyserver to be running.
with HTTPConnectionPool(self.host, self.port) as pool:
# Get the HTTPConnection instance
conn = pool._new_conn()
try:
# Update the default socket options
conn.default_socket_options += [(socket.SOL_SOCKET,
socket.SO_KEEPALIVE, 1)]
conn.connect()
s = conn._sock
nagle_disabled = (s.getsockopt(socket.IPPROTO_TCP,
socket.TCP_NODELAY) > 0)
using_keepalive = (s.getsockopt(socket.SOL_SOCKET,
socket.SO_KEEPALIVE) > 0)
assert nagle_disabled
assert using_keepalive
finally:
conn.close()
s.close()
def test_queue_monkeypatching(self):
with mock.patch.object(queue, "Empty", BadError):
with HTTPConnectionPool(host="localhost", block=True) as http:
http._get_conn()
with pytest.raises(EmptyPoolError):
http._get_conn(timeout=0)
def test_mixed_case_hostname(self):
with HTTPConnectionPool("LoCaLhOsT", self.port) as pool:
response = pool.request("GET", "http://LoCaLhOsT:%d/" % self.port)
assert response.status == 200
