def test_incomplete_response(self):
body = 'Response'
partial_body = body[:2]
def socket_handler(listener):
sock = listener.accept()[0]
# Consume request
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65536)
# Send partial response and close socket.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), partial_body)).encode('utf-8'))
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
response = yield From(
pool.request('GET', '/', retries=0, preload_content=False))
self.aioAssertRaises(ProtocolError, response.read)
def test_delayed_body_read_timeout(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
buf = b''
body = 'Hi'
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65536)
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n' % len(body)).encode('utf-8'))
timed_out.wait()
sock.send(body.encode('utf-8'))
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
response = yield From(pool.urlopen('GET', '/', retries=0, preload_content=False,
timeout=Timeout(connect=1, read=0.1)))
try:
self.aioAssertRaises(ReadTimeoutError, response.read)
finally:
timed_out.set()
def tst_bad_connect(self):
pool = HTTPConnectionPool("badhost.invalid", self.port)
try:
yield From(pool.request("GET", "/", retries=5))
self.fail("should raise timeout exception here")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError), e.reason)
def test_delayed_body_read_timeout(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
buf = b''
body = 'Hi'
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65536)
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n' % len(body)).encode('utf-8'))
timed_out.wait()
sock.send(body.encode('utf-8'))
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
response = yield From(
pool.urlopen('GET',
'/',
retries=0,
preload_content=False,
timeout=Timeout(connect=1, read=0.1)))
try:
self.aioAssertRaises(ReadTimeoutError, response.read)
finally:
timed_out.set()
def tst_bad_connect(self):
pool = HTTPConnectionPool('badhost.invalid', self.port)
try:
yield From(pool.request('GET', '/', retries=5))
self.fail("should raise timeout exception here")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError), e.reason)
def test_incomplete_response(self):
body = 'Response'
partial_body = body[:2]
def socket_handler(listener):
sock = listener.accept()[0]
# Consume request
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65536)
# Send partial response and close socket.
sock.send((
'HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), partial_body)).encode('utf-8')
)
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
response = yield From(pool.request('GET', '/', retries=0, preload_content=False))
self.aioAssertRaises(ProtocolError, response.read)
def test_source_address(self):
for addr in VALID_SOURCE_ADDRESSES:
pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False)
r = yield From(pool.request('GET', '/source_address'))
assert (yield From(r.data) == b(addr[0]), (
"expected the response to contain the source address {addr}, "
"but was {data}".format(data=(yield From(r.data)), addr=b(addr[0]))))
def test_release_conn_parameter(self):
MAXSIZE = 5
pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
self.assertEqual(pool.pool.qsize(), MAXSIZE)
# Make request without releasing connection
yield From(pool.request("GET", "/", release_conn=False, preload_content=False))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
def test_release_conn_parameter(self):
MAXSIZE=5
pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
self.assertEqual(pool.pool.qsize(), MAXSIZE)
# Make request without releasing connection
yield From(pool.request('GET', '/', release_conn=False, preload_content=False))
self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
def test_keepalive(self):
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1)
r = yield From(pool.request('GET', '/keepalive?close=0'))
r = yield From(pool.request('GET', '/keepalive?close=0'))
self.assertEqual(r.status, 200)
self.assertEqual(pool.num_connections, 1)
self.assertEqual(pool.num_requests, 2)
def tst_socket_options(self):
"""Test that connections accept socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)])
s = pool._new_conn()._new_conn() # Get the socket
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(using_keepalive)
s.close()
def test_connection_count_bigpool(self):
http_pool = HTTPConnectionPool(self.host, self.port, maxsize=16)
yield From(http_pool.request("GET", "/"))
yield From(http_pool.request("GET", "/"))
yield From(http_pool.request("GET", "/"))
self.assertEqual(http_pool.num_connections, 1)
self.assertEqual(http_pool.num_requests, 3)
def tst_disable_default_socket_options(self):
"""Test that passing None disables all socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=None)
s = pool._new_conn()._new_conn()
using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0
self.assertTrue(using_nagle)
s.close()
def test_keepalive(self):
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1)
r = yield From(pool.request("GET", "/keepalive?close=0"))
r = yield From(pool.request("GET", "/keepalive?close=0"))
self.assertEqual(r.status, 200)
self.assertEqual(pool.num_connections, 1)
self.assertEqual(pool.num_requests, 2)
def tst_disable_default_socket_options(self):
"""Test that passing None disables all socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=None)
s = pool._new_conn()._new_conn()
using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0
self.assertTrue(using_nagle)
s.close()
def test_connection_count_bigpool(self):
http_pool = HTTPConnectionPool(self.host, self.port, maxsize=16)
yield From(http_pool.request('GET', '/'))
yield From(http_pool.request('GET', '/'))
yield From(http_pool.request('GET', '/'))
self.assertEqual(http_pool.num_connections, 1)
self.assertEqual(http_pool.num_requests, 3)
def test_timeout_reset(self):
""" If the read timeout isn't set, socket timeout should reset """
url = '/sleep?seconds=0.005'
timeout = Timeout(connect=0.001)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
try:
yield From(pool._make_request(conn, 'GET', url))
except ReadTimeoutError:
self.fail("This request shouldn't trigger a read timeout.")
def test_partial_response(self):
pool = HTTPConnectionPool(self.host, self.port, maxsize=1)
req_data = {"lol": "cat"}
resp_data = urlencode(req_data).encode("utf-8")
r = yield From(pool.request("GET", "/echo", fields=req_data, preload_content=False))
self.assertEqual(r.read(5), resp_data[:5])
self.assertEqual(r.read(), resp_data[5:])
def test_partial_response(self):
pool = HTTPConnectionPool(self.host, self.port, maxsize=1)
req_data = {'lol': 'cat'}
resp_data = urlencode(req_data).encode('utf-8')
r = yield From(pool.request('GET', '/echo', fields=req_data, preload_content=False))
self.assertEqual(r.read(5), resp_data[:5])
self.assertEqual(r.read(), resp_data[5:])
def test_connection_error_retries(self):
""" ECONNREFUSED error should raise a connection error, with retries """
port = find_unused_port()
pool = HTTPConnectionPool(self.host, port)
try:
yield From(pool.request("GET", "/", retries=Retry(connect=3)))
self.fail("Should have failed with a connection error.")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError))
self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED)
def test_timeout_reset(self):
""" If the read timeout isn't set, socket timeout should reset """
url = "/sleep?seconds=0.005"
timeout = Timeout(connect=0.001)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
try:
yield From(pool._make_request(conn, "GET", url))
except ReadTimeoutError:
self.fail("This request shouldn't trigger a read timeout.")
def test_connection_error_retries(self):
""" ECONNREFUSED error should raise a connection error, with retries """
port = find_unused_port()
pool = HTTPConnectionPool(self.host, port)
try:
yield From(pool.request('GET', '/', retries=Retry(connect=3)))
self.fail("Should have failed with a connection error.")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError))
self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED)
def test_source_address(self):
for addr in VALID_SOURCE_ADDRESSES:
pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False)
r = yield From(pool.request("GET", "/source_address"))
assert (
yield From(r.data) == b(addr[0]),
(
"expected the response to contain the source address {addr}, "
"but was {data}".format(data=(yield From(r.data)), addr=b(addr[0]))
),
)
def tst_socket_options(self):
"""Test that connections accept socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=[
(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
])
s = pool._new_conn()._new_conn() # Get the socket
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(using_keepalive)
s.close()
def test_nagle(self):
""" Test that connections have TCP_NODELAY turned on """
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
conn = pool._get_conn()
yield From(pool._make_request(conn, 'GET', '/'))
tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
assert tcp_nodelay_setting > 0, ("Expected TCP_NODELAY to be set on the "
"socket (with value greater than 0) "
"but instead was %s" %
tcp_nodelay_setting)
def test_nagle(self):
""" Test that connections have TCP_NODELAY turned on """
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
conn = pool._get_conn()
yield From(pool._make_request(conn, "GET", "/"))
tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
assert tcp_nodelay_setting > 0, (
"Expected TCP_NODELAY to be set on the "
"socket (with value greater than 0) "
"but instead was %s" % tcp_nodelay_setting
)
def tst_defaults_are_applied(self):
"""Test that modifying the default socket options works."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
# Get the HTTPConnection instance
conn = pool._new_conn()
# Update the default socket options
conn.default_socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
s = conn._new_conn()
nagle_disabled = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(nagle_disabled)
self.assertTrue(using_keepalive)
def tst_defaults_are_applied(self):
"""Test that modifying the default socket options works."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
# Get the HTTPConnection instance
conn = pool._new_conn()
# Update the default socket options
conn.default_socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
s = conn._new_conn()
nagle_disabled = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(nagle_disabled)
self.assertTrue(using_keepalive)
def test_connect_timeout(self):
url = '/sleep?seconds=0.005'
timeout = Timeout(connect=0.001)
# Pool-global timeout
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url))
# Retries
retries = Retry(connect=0)
yield From(self.aioAssertRaises(MaxRetryError, pool.request, 'GET', url,
retries=retries))
# Request-specific connection timeouts
big_timeout = Timeout(read=0.2, connect=0.2)
pool = HTTPConnectionPool(TARPIT_HOST, self.port,
timeout=big_timeout, retries=False)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET',
url, timeout=timeout))
pool._put_conn(conn)
yield From(self.aioAssertRaises(ConnectTimeoutError, pool.request, 'GET', url,
timeout=timeout))
def test_timeout_success(self):
timeout = Timeout(connect=3, read=5, total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
yield From(pool.request('GET', '/'))
# This should not raise a "Timeout already started" error
yield From(pool.request('GET', '/'))
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
# This should also not raise a "Timeout already started" error
yield From(pool.request('GET', '/'))
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
yield From(pool.request('GET', '/'))
def test_lazy_load_twice(self):
# This test is sad and confusing. Need to figure out what's
# going on with partial reads and socket reuse.
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
payload_size = 1024 * 2
first_chunk = 512
boundary = "foo"
req_data = {"count": "a" * payload_size}
resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0]
req2_data = {"count": "b" * payload_size}
resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0]
r1 = yield From(
pool.request("POST", "/echo", fields=req_data, multipart_boundary=boundary, preload_content=False)
)
self.assertEqual(r1.read(first_chunk), resp_data[:first_chunk])
try:
r2 = yield From(
pool.request(
"POST",
"/echo",
fields=req2_data,
multipart_boundary=boundary,
preload_content=False,
pool_timeout=0.001,
)
)
# This branch should generally bail here, but maybe someday it will
# work? Perhaps by some sort of magic. Consider it a TODO.
self.assertEqual(r2.read(first_chunk), resp2_data[:first_chunk])
self.assertEqual((yield From(r1.read()), resp_data[first_chunk:]))
self.assertEqual((yield From(r2.read()), resp2_data[first_chunk:]))
self.assertEqual(pool.num_requests, 2)
except EmptyPoolError:
self.assertEqual((yield From(r1.read()), resp_data[first_chunk:]))
self.assertEqual(pool.num_requests, 1)
self.assertEqual(pool.num_connections, 1)
def test_unknown_protocol(self):
handler = create_response_handler(b'HTTP/1000 200 OK\r\n'
b'Content-Length: 0\r\n'
b'\r\n')
self._start_server(handler)
pool = HTTPConnectionPool(self.host, self.port, retries=False)
self.aioAssertRaises(ProtocolError, pool.request, 'GET', '/')
def test_bad_statusline(self):
handler = create_response_handler(b'HTTP/1.1 Omg What Is This?\r\n'
b'Content-Length: 0\r\n'
b'\r\n')
self._start_server(handler)
pool = HTTPConnectionPool(self.host, self.port, retries=False)
self.aioAssertRaises(ProtocolError, pool.request, 'GET', '/')
def test_for_double_release(self):
MAXSIZE=5
# Check default state
pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
self.assertEqual(pool.num_connections, 0)
self.assertEqual(pool.pool.qsize(), MAXSIZE)
# Make an empty slot for testing
pool.pool.get()
self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
# Check state after simple request
yield From(pool.urlopen('GET', '/'))
self.assertEqual(pool.pool.qsize(), MAXSIZE-1)
# Check state without release
yield From(pool.urlopen('GET', '/', preload_content=False))
self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
yield From(pool.urlopen('GET', '/'))
self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
# Check state after read
yield From(pool.urlopen('GET', '/').data)
self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
yield From(pool.urlopen('GET', '/'))
self.assertEqual(pool.pool.qsize(), MAXSIZE-2)
def test_connection_refused(self):
# Does the pool retry if there is no listener on the port?
host, port = get_unreachable_address()
pool = HTTPConnectionPool(host, port)
self.aioAssertRaises(MaxRetryError,
pool.request,
'GET',
'/',
retries=0)
def test_multi_setcookie(self):
def multicookie_response_handler(listener):
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
sock.send(b'HTTP/1.1 200 OK\r\n'
b'Set-Cookie: foo=1\r\n'
b'Set-Cookie: bar=1\r\n'
b'\r\n')
sock.close()
self._start_server(multicookie_response_handler)
pool = HTTPConnectionPool(self.host, self.port)
r = yield From(pool.request('GET', '/', retries=0))
self.assertEqual(r.headers, {'set-cookie': 'foo=1, bar=1'})
def test_multi_setcookie(self):
def multicookie_response_handler(listener):
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
sock.send(b'HTTP/1.1 200 OK\r\n'
b'Set-Cookie: foo=1\r\n'
b'Set-Cookie: bar=1\r\n'
b'\r\n')
sock.close()
self._start_server(multicookie_response_handler)
pool = HTTPConnectionPool(self.host, self.port)
r = yield From(pool.request('GET', '/', retries=0))
self.assertEqual(r.headers, {'set-cookie': 'foo=1, bar=1'})
def test_conn_closed(self):
pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
conn = pool._get_conn()
pool._put_conn(conn)
try:
url = '/sleep?seconds=0.005'
yield From(pool.urlopen('GET', url))
self.fail("The request should fail with a timeout error.")
except ReadTimeoutError:
if conn.sock:
self.assertRaises(socket.error, conn.sock.recv, 1024)
finally:
pool._put_conn(conn)
def test_retry_weird_http_version(self):
""" Retry class should handle httplib.BadStatusLine errors properly """
def socket_handler(listener):
sock = listener.accept()[0]
# First request.
# Pause before responding so the first request times out.
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# send unknown http protocol
body = "bad http 0.5 response"
sock.send(('HTTP/0.5 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close()
# Second request.
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# Now respond immediately.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'foo' % (len('foo'))).encode('utf-8'))
sock.close() # Close the socket.
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
retry = Retry(read=1)
response = yield From(pool.request('GET', '/', retries=retry))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'foo'))
def tst_timeout_errors_cause_retries(self):
def socket_handler(listener):
sock_timeout = listener.accept()[0]
# Wait for a second request before closing the first socket.
sock = listener.accept()[0]
sock_timeout.close()
# Second request.
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# Now respond immediately.
body = 'Response 2'
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close()
# In situations where the main thread throws an exception, the server
# thread can hang on an accept() call. This ensures everything times
# out within 1 second. This should be long enough for any socket
# operations in the test suite to complete
default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(1)
try:
self._start_server(socket_handler)
t = Timeout(connect=0.1, read=0.1)
pool = HTTPConnectionPool(self.host, self.port, timeout=t)
response = yield From(pool.request('GET', '/', retries=1))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'Response 2'))
finally:
socket.setdefaulttimeout(default_timeout)
def test_for_double_release(self):
MAXSIZE = 5
# Check default state
pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
self.assertEqual(pool.num_connections, 0)
self.assertEqual(pool.pool.qsize(), MAXSIZE)
# Make an empty slot for testing
pool.pool.get()
self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
# Check state after simple request
yield From(pool.urlopen("GET", "/"))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
# Check state without release
yield From(pool.urlopen("GET", "/", preload_content=False))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
yield From(pool.urlopen("GET", "/"))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
# Check state after read
yield From(pool.urlopen("GET", "/").data)
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
yield From(pool.urlopen("GET", "/"))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
def test_retry_weird_http_version(self):
""" Retry class should handle httplib.BadStatusLine errors properly """
def socket_handler(listener):
sock = listener.accept()[0]
# First request.
# Pause before responding so the first request times out.
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# send unknown http protocol
body = "bad http 0.5 response"
sock.send(('HTTP/0.5 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close()
# Second request.
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# Now respond immediately.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'foo' % (len('foo'))).encode('utf-8'))
sock.close() # Close the socket.
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
retry = Retry(read=1)
response = yield From(pool.request('GET', '/', retries=retry))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'foo'))
def tst_timeout_errors_cause_retries(self):
def socket_handler(listener):
sock_timeout = listener.accept()[0]
# Wait for a second request before closing the first socket.
sock = listener.accept()[0]
sock_timeout.close()
# Second request.
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# Now respond immediately.
body = 'Response 2'
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close()
# In situations where the main thread throws an exception, the server
# thread can hang on an accept() call. This ensures everything times
# out within 1 second. This should be long enough for any socket
# operations in the test suite to complete
default_timeout = socket.getdefaulttimeout()
socket.setdefaulttimeout(1)
try:
self._start_server(socket_handler)
t = Timeout(connect=0.1, read=0.1)
pool = HTTPConnectionPool(self.host, self.port, timeout=t)
response = yield From(pool.request('GET', '/', retries=1))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'Response 2'))
finally:
socket.setdefaulttimeout(default_timeout)
def test_lazy_load_twice(self):
# This test is sad and confusing. Need to figure out what's
# going on with partial reads and socket reuse.
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
payload_size = 1024 * 2
first_chunk = 512
boundary = 'foo'
req_data = {'count': 'a' * payload_size}
resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0]
req2_data = {'count': 'b' * payload_size}
resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0]
r1 = yield From(pool.request('POST', '/echo', fields=req_data, multipart_boundary=boundary, preload_content=False))
self.assertEqual(r1.read(first_chunk), resp_data[:first_chunk])
try:
r2 = yield From(pool.request('POST', '/echo', fields=req2_data, multipart_boundary=boundary,
preload_content=False, pool_timeout=0.001))
# This branch should generally bail here, but maybe someday it will
# work? Perhaps by some sort of magic. Consider it a TODO.
self.assertEqual(r2.read(first_chunk), resp2_data[:first_chunk])
self.assertEqual((yield From(r1.read()), resp_data[first_chunk:]))
self.assertEqual((yield From(r2.read()), resp2_data[first_chunk:]))
self.assertEqual(pool.num_requests, 2)
except EmptyPoolError:
self.assertEqual((yield From(r1.read()), resp_data[first_chunk:]))
self.assertEqual(pool.num_requests, 1)
self.assertEqual(pool.num_connections, 1)
def test_total_timeout(self):
url = '/sleep?seconds=0.005'
timeout = Timeout(connect=3, read=5, total=0.001)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url))
# This will get the socket to raise an EAGAIN on the read
timeout = Timeout(connect=3, read=0)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url))
# The connect should succeed and this should hit the read timeout
timeout = Timeout(connect=3, read=5, total=0.002)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, 'GET', url))
def tst_recovery_when_server_closes_connection(self):
# Does the pool work seamlessly if an open connection in the
# connection pool gets hung up on by the server, then reaches
# the front of the queue again?
done_closing = Event()
def socket_handler(listener):
for i in 0, 1:
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65536)
body = 'Response %d' % i
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close() # simulate a server timing out, closing socket
done_closing.set() # let the test know it can proceed
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
response = yield From(pool.request('GET', '/', retries=0))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'Response 0'))
done_closing.wait() # wait until the socket in our pool gets closed
response = yield From(pool.request('GET', '/', retries=0))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'Response 1'))
def tst_recovery_when_server_closes_connection(self):
# Does the pool work seamlessly if an open connection in the
# connection pool gets hung up on by the server, then reaches
# the front of the queue again?
done_closing = Event()
def socket_handler(listener):
for i in 0, 1:
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf = sock.recv(65536)
body = 'Response %d' % i
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close() # simulate a server timing out, closing socket
done_closing.set() # let the test know it can proceed
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
response = yield From(pool.request('GET', '/', retries=0))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'Response 0'))
done_closing.wait() # wait until the socket in our pool gets closed
response = yield From(pool.request('GET', '/', retries=0))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'Response 1'))
def test_conn_closed(self):
pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
conn = pool._get_conn()
pool._put_conn(conn)
try:
url = "/sleep?seconds=0.005"
yield From(pool.urlopen("GET", url))
self.fail("The request should fail with a timeout error.")
except ReadTimeoutError:
if conn.sock:
self.assertRaises(socket.error, conn.sock.recv, 1024)
finally:
pool._put_conn(conn)
def test_timeout_success(self):
timeout = Timeout(connect=3, read=5, total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
yield From(pool.request("GET", "/"))
# This should not raise a "Timeout already started" error
yield From(pool.request("GET", "/"))
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
# This should also not raise a "Timeout already started" error
yield From(pool.request("GET", "/"))
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
yield From(pool.request("GET", "/"))
def test_tunnel(self):
# note the actual httplib.py has no tests for this functionality
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
try:
conn.set_tunnel(self.host, self.port)
except AttributeError: # python 2.6
conn._set_tunnel(self.host, self.port)
conn._tunnel = mock.Mock(return_value=None)
yield From(pool._make_request(conn, 'GET', '/'))
conn._tunnel.assert_called_once_with()
# test that it's not called when tunnel is not set
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
conn._tunnel = mock.Mock(return_value=None)
yield From(pool._make_request(conn, 'GET', '/'))
self.assertEqual(conn._tunnel.called, False)
def test_connection_read_timeout(self):
timed_out = Event()
def socket_handler(listener):
sock = listener.accept()[0]
while not sock.recv(65536).endswith(b'\r\n\r\n'):
pass
timed_out.wait()
sock.close()
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host,
self.port,
timeout=0.001,
retries=False)
try:
self.aioAssertRaises(ReadTimeoutError, pool.request, 'GET', '/')
finally:
timed_out.set()
def test_total_timeout(self):
url = "/sleep?seconds=0.005"
timeout = Timeout(connect=3, read=5, total=0.001)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url))
# This will get the socket to raise an EAGAIN on the read
timeout = Timeout(connect=3, read=0)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url))
# The connect should succeed and this should hit the read timeout
timeout = Timeout(connect=3, read=5, total=0.002)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url))
def test_connect_timeout(self):
url = "/sleep?seconds=0.005"
timeout = Timeout(connect=0.001)
# Pool-global timeout
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url))
# Retries
retries = Retry(connect=0)
yield From(self.aioAssertRaises(MaxRetryError, pool.request, "GET", url, retries=retries))
# Request-specific connection timeouts
big_timeout = Timeout(read=0.2, connect=0.2)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=big_timeout, retries=False)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url, timeout=timeout))
pool._put_conn(conn)
yield From(self.aioAssertRaises(ConnectTimeoutError, pool.request, "GET", url, timeout=timeout))
def test_tunnel(self):
# note the actual httplib.py has no tests for this functionality
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
try:
conn.set_tunnel(self.host, self.port)
except AttributeError: # python 2.6
conn._set_tunnel(self.host, self.port)
conn._tunnel = mock.Mock(return_value=None)
yield From(pool._make_request(conn, "GET", "/"))
conn._tunnel.assert_called_once_with()
# test that it's not called when tunnel is not set
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
conn._tunnel = mock.Mock(return_value=None)
yield From(pool._make_request(conn, "GET", "/"))
self.assertEqual(conn._tunnel.called, False)
def test_none_total_applies_connect(self):
url = "/sleep?seconds=0.005"
timeout = Timeout(total=None, connect=0.001)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url))
def setUp(self):
self.pool = HTTPConnectionPool(self.host, self.port)
def setUp(self):
self.pool = HTTPConnectionPool(self.host, self.port)
def test_keepalive_close(self):
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
r = yield From(pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}))
self.assertEqual(pool.num_connections, 1)
# The dummyserver will have responded with Connection:close,
# and httplib will properly cleanup the socket.
# We grab the HTTPConnection object straight from the Queue,
# because _get_conn() is where the check & reset occurs
# pylint: disable-msg=W0212
conn = pool.pool.get()
self.assertEqual(conn.sock, None)
pool._put_conn(conn)
# Now with keep-alive
r = yield From(pool.request("GET", "/keepalive?close=0", retries=0, headers={"Connection": "keep-alive"}))
# The dummyserver responded with Connection:keep-alive, the connection
# persists.
conn = pool.pool.get()
self.assertNotEqual(conn.sock, None)
pool._put_conn(conn)
# Another request asking the server to close the connection. This one
# should get cleaned up for the next request.
r = yield From(pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}))
self.assertEqual(r.status, 200)
conn = pool.pool.get()
self.assertEqual(conn.sock, None)
pool._put_conn(conn)
# Next request
r = yield From(pool.request("GET", "/keepalive?close=0"))
class TestRetry(HTTPDummyServerTestCase):
def setUp(self):
self.pool = HTTPConnectionPool(self.host, self.port)
@asyncio.coroutine
def test_max_retry(self):
try:
r = yield From(self.pool.request("GET", "/redirect", fields={"target": "/"}, retries=0))
self.fail("Failed to raise MaxRetryError exception, returned %r" % r.status)
except MaxRetryError:
pass
@asyncio.coroutine
def test_disabled_retry(self):
""" Disabled retries should disable redirect handling. """
r = yield From(self.pool.request("GET", "/redirect", fields={"target": "/"}, retries=False))
self.assertEqual(r.status, 303)
r = yield From(self.pool.request("GET", "/redirect", fields={"target": "/"}, retries=Retry(redirect=False)))
self.assertEqual(r.status, 303)
# don't pass for orig
# pool = HTTPConnectionPool('thishostdoesnotexist.invalid', self.port, timeout=0.001)
# self.assertRaises(ProtocolError, pool.request, 'GET', '/test', retries=False)
@asyncio.coroutine
def test_read_retries(self):
""" Should retry for status codes in the whitelist """
retry = Retry(read=1, status_forcelist=[418])
resp = yield From(
self.pool.request("GET", "/successful_retry", headers={"test-name": "test_read_retries"}, retries=retry)
)
self.assertEqual(resp.status, 200)
@asyncio.coroutine
def test_read_total_retries(self):
""" HTTP response w/ status code in the whitelist should be retried """
headers = {"test-name": "test_read_total_retries"}
retry = Retry(total=1, status_forcelist=[418])
resp = yield From(self.pool.request("GET", "/successful_retry", headers=headers, retries=retry))
self.assertEqual(resp.status, 200)
@asyncio.coroutine
def test_retries_wrong_whitelist(self):
"""HTTP response w/ status code not in whitelist shouldn't be retried"""
retry = Retry(total=1, status_forcelist=[202])
resp = yield From(
self.pool.request("GET", "/successful_retry", headers={"test-name": "test_wrong_whitelist"}, retries=retry)
)
self.assertEqual(resp.status, 418)
@asyncio.coroutine
def test_default_method_whitelist_retried(self):
""" urllib3 should retry methods in the default method whitelist """
retry = Retry(total=1, status_forcelist=[418])
resp = yield From(
self.pool.request(
"OPTIONS", "/successful_retry", headers={"test-name": "test_default_whitelist"}, retries=retry
)
)
self.assertEqual(resp.status, 200)
@asyncio.coroutine
def test_retries_wrong_method_list(self):
"""Method not in our whitelist should not be retried, even if code matches"""
headers = {"test-name": "test_wrong_method_whitelist"}
retry = Retry(total=1, status_forcelist=[418], method_whitelist=["POST"])
resp = yield From(self.pool.request("GET", "/successful_retry", headers=headers, retries=retry))
self.assertEqual(resp.status, 418)
@asyncio.coroutine
def test_read_retries_unsuccessful(self):
headers = {"test-name": "test_read_retries_unsuccessful"}
resp = yield From(self.pool.request("GET", "/successful_retry", headers=headers, retries=1))
self.assertEqual(resp.status, 418)
@asyncio.coroutine
def test_retry_reuse_safe(self):
""" It should be possible to reuse a Retry object across requests """
headers = {"test-name": "test_retry_safe"}
retry = Retry(total=1, status_forcelist=[418])
resp = yield From(self.pool.request("GET", "/successful_retry", headers=headers, retries=retry))
self.assertEqual(resp.status, 200)
resp = yield From(self.pool.request("GET", "/successful_retry", headers=headers, retries=retry))
self.assertEqual(resp.status, 200)
class TestConnectionPool(HTTPDummyServerTestCase):
@asyncio.coroutine
def aioAssertRaises(self, exc, f, *args, **kwargs):
"""tests a coroutine for whether it raises given error."""
try:
yield From(f(*args, **kwargs))
except exc as e:
pass
else:
self.fail("expected %s not raised" % exc.__name__)
def setUp(self):
self.pool = HTTPConnectionPool(self.host, self.port)
@asyncio.coroutine
def get(self):
r = yield From(self.pool.request("GET", "/specific_method", fields={"method": "GET"}))
self.assertEqual(r.status, 200, (yield From(r.data)))
@asyncio.coroutine
def post_url(self):
r = yield From(self.pool.request("POST", "/specific_method", fields={"method": "POST"}))
self.assertEqual(r.status, 200, (yield From(r.data)))
@asyncio.coroutine
def test_urlopen_put(self):
r = yield From(self.pool.urlopen("PUT", "/specific_method?method=PUT"))
self.assertEqual(r.status, 200, (yield From(r.data)))
@asyncio.coroutine
def test_wrong_specific_method(self):
# To make sure the dummy server is actually returning failed responses
r = yield From(self.pool.request("GET", "/specific_method", fields={"method": "POST"}))
self.assertEqual(r.status, 400, (yield From(r.data)))
r = yield From(self.pool.request("POST", "/specific_method", fields={"method": "GET"}))
self.assertEqual(r.status, 400, (yield From(r.data)))
@asyncio.coroutine
def test_upload(self):
data = "I'm in ur multipart form-data, hazing a cheezburgr"
fields = {
"upload_param": "filefield",
"upload_filename": "lolcat.txt",
"upload_size": len(data),
"filefield": ("lolcat.txt", data),
}
r = yield From(self.pool.request("POST", "/upload", fields=fields))
self.assertEqual(r.status, 200, (yield From(r.data)))
@asyncio.coroutine
def test_one_name_multiple_values(self):
fields = [("foo", "a"), ("foo", "b")]
# urlencode
r = yield From(self.pool.request("GET", "/echo", fields=fields))
self.assertEqual((yield From(r.data)), b"foo=a&foo=b")
# multipart
r = yield From(self.pool.request("POST", "/echo", fields=fields))
self.assertEqual((yield From(r.data).count(b'name="foo"'), 2))
@asyncio.coroutine
def test_unicode_upload(self):
fieldname = u("myfile")
filename = u("\xe2\x99\xa5.txt")
data = u("\xe2\x99\xa5").encode("utf8")
size = len(data)
fields = {
u("upload_param"): fieldname,
u("upload_filename"): filename,
u("upload_size"): size,
fieldname: (filename, data),
}
r = yield From(self.pool.request("POST", "/upload", fields=fields))
self.assertEqual(r.status, 200, (yield From(r.data)))
@asyncio.coroutine
def test_timeout_float(self):
url = "/sleep?seconds=0.005"
# Pool-global timeout
pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
yield From(self.aioAssertRaises(ReadTimeoutError, pool.request, "GET", url))
@asyncio.coroutine
def test_conn_closed(self):
pool = HTTPConnectionPool(self.host, self.port, timeout=0.001, retries=False)
conn = pool._get_conn()
pool._put_conn(conn)
try:
url = "/sleep?seconds=0.005"
yield From(pool.urlopen("GET", url))
self.fail("The request should fail with a timeout error.")
except ReadTimeoutError:
if conn.sock:
self.assertRaises(socket.error, conn.sock.recv, 1024)
finally:
pool._put_conn(conn)
@asyncio.coroutine
def test_nagle(self):
""" Test that connections have TCP_NODELAY turned on """
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
conn = pool._get_conn()
yield From(pool._make_request(conn, "GET", "/"))
tcp_nodelay_setting = conn.sock.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
assert tcp_nodelay_setting > 0, (
"Expected TCP_NODELAY to be set on the "
"socket (with value greater than 0) "
"but instead was %s" % tcp_nodelay_setting
)
def tst_socket_options(self):
"""Test that connections accept socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=[(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)])
s = pool._new_conn()._new_conn() # Get the socket
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(using_keepalive)
s.close()
def tst_disable_default_socket_options(self):
"""Test that passing None disables all socket options."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port, socket_options=None)
s = pool._new_conn()._new_conn()
using_nagle = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) == 0
self.assertTrue(using_nagle)
s.close()
def tst_defaults_are_applied(self):
"""Test that modifying the default socket options works."""
# This test needs to be here in order to be run. socket.create_connection actually tries to
# connect to the host provided so we need a dummyserver to be running.
pool = HTTPConnectionPool(self.host, self.port)
# Get the HTTPConnection instance
conn = pool._new_conn()
# Update the default socket options
conn.default_socket_options += [(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)]
s = conn._new_conn()
nagle_disabled = s.getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY) > 0
using_keepalive = s.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE) > 0
self.assertTrue(nagle_disabled)
self.assertTrue(using_keepalive)
@timed(0.5)
@asyncio.coroutine
def test_timeout(self):
""" Requests should time out when expected """
url = "/sleep?seconds=0.002"
timeout = Timeout(read=0.001)
# Pool-global timeout
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout, retries=False)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url))
pool._put_conn(conn)
time.sleep(0.02) # Wait for server to start receiving again. :(
yield From(self.aioAssertRaises(ReadTimeoutError, pool.request, "GET", url))
# Request-specific timeouts should raise errors
pool = HTTPConnectionPool(self.host, self.port, timeout=0.1, retries=False)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url, timeout=timeout))
pool._put_conn(conn)
time.sleep(0.02) # Wait for server to start receiving again. :(
yield From(self.aioAssertRaises(ReadTimeoutError, pool.request, "GET", url, timeout=timeout))
# Timeout int/float passed directly to request and _make_request should
# raise a request timeout
yield From(self.aioAssertRaises(ReadTimeoutError, pool.request, "GET", url, timeout=0.001))
conn = pool._new_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url, timeout=0.001))
pool._put_conn(conn)
# Timeout int/float passed directly to _make_request should not raise a
# request timeout if it's a high value
yield From(pool.request("GET", url, timeout=1))
@requires_network
@timed(0.5)
@asyncio.coroutine
def test_connect_timeout(self):
url = "/sleep?seconds=0.005"
timeout = Timeout(connect=0.001)
# Pool-global timeout
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url))
# Retries
retries = Retry(connect=0)
yield From(self.aioAssertRaises(MaxRetryError, pool.request, "GET", url, retries=retries))
# Request-specific connection timeouts
big_timeout = Timeout(read=0.2, connect=0.2)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=big_timeout, retries=False)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url, timeout=timeout))
pool._put_conn(conn)
yield From(self.aioAssertRaises(ConnectTimeoutError, pool.request, "GET", url, timeout=timeout))
@asyncio.coroutine
def test_connection_error_retries(self):
""" ECONNREFUSED error should raise a connection error, with retries """
port = find_unused_port()
pool = HTTPConnectionPool(self.host, port)
try:
yield From(pool.request("GET", "/", retries=Retry(connect=3)))
self.fail("Should have failed with a connection error.")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError))
self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED)
@asyncio.coroutine
def test_timeout_reset(self):
""" If the read timeout isn't set, socket timeout should reset """
url = "/sleep?seconds=0.005"
timeout = Timeout(connect=0.001)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
try:
yield From(pool._make_request(conn, "GET", url))
except ReadTimeoutError:
self.fail("This request shouldn't trigger a read timeout.")
@requires_network
@timed(5.0)
@asyncio.coroutine
def test_total_timeout(self):
url = "/sleep?seconds=0.005"
timeout = Timeout(connect=3, read=5, total=0.001)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url))
# This will get the socket to raise an EAGAIN on the read
timeout = Timeout(connect=3, read=0)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url))
# The connect should succeed and this should hit the read timeout
timeout = Timeout(connect=3, read=5, total=0.002)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ReadTimeoutError, pool._make_request, conn, "GET", url))
@requires_network
@asyncio.coroutine
def test_none_total_applies_connect(self):
url = "/sleep?seconds=0.005"
timeout = Timeout(total=None, connect=0.001)
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, "GET", url))
@asyncio.coroutine
def test_timeout_success(self):
timeout = Timeout(connect=3, read=5, total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
yield From(pool.request("GET", "/"))
# This should not raise a "Timeout already started" error
yield From(pool.request("GET", "/"))
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
# This should also not raise a "Timeout already started" error
yield From(pool.request("GET", "/"))
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
yield From(pool.request("GET", "/"))
@asyncio.coroutine
def test_tunnel(self):
# note the actual httplib.py has no tests for this functionality
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
try:
conn.set_tunnel(self.host, self.port)
except AttributeError: # python 2.6
conn._set_tunnel(self.host, self.port)
conn._tunnel = mock.Mock(return_value=None)
yield From(pool._make_request(conn, "GET", "/"))
conn._tunnel.assert_called_once_with()
# test that it's not called when tunnel is not set
timeout = Timeout(total=None)
pool = HTTPConnectionPool(self.host, self.port, timeout=timeout)
conn = pool._get_conn()
conn._tunnel = mock.Mock(return_value=None)
yield From(pool._make_request(conn, "GET", "/"))
self.assertEqual(conn._tunnel.called, False)
@asyncio.coroutine
def test_redirect(self):
r = yield From(self.pool.request("GET", "/redirect", fields={"target": "/"}, redirect=False))
self.assertEqual(r.status, 303)
r = yield From(self.pool.request("GET", "/redirect", fields={"target": "/"}))
self.assertEqual(r.status, 200)
self.assertEqual((yield From(r.data)), b"Dummy server!")
def tst_bad_connect(self):
pool = HTTPConnectionPool("badhost.invalid", self.port)
try:
yield From(pool.request("GET", "/", retries=5))
self.fail("should raise timeout exception here")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError), e.reason)
@asyncio.coroutine
def test_keepalive(self):
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1)
r = yield From(pool.request("GET", "/keepalive?close=0"))
r = yield From(pool.request("GET", "/keepalive?close=0"))
self.assertEqual(r.status, 200)
self.assertEqual(pool.num_connections, 1)
self.assertEqual(pool.num_requests, 2)
@asyncio.coroutine
def test_keepalive_close(self):
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
r = yield From(pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}))
self.assertEqual(pool.num_connections, 1)
# The dummyserver will have responded with Connection:close,
# and httplib will properly cleanup the socket.
# We grab the HTTPConnection object straight from the Queue,
# because _get_conn() is where the check & reset occurs
# pylint: disable-msg=W0212
conn = pool.pool.get()
self.assertEqual(conn.sock, None)
pool._put_conn(conn)
# Now with keep-alive
r = yield From(pool.request("GET", "/keepalive?close=0", retries=0, headers={"Connection": "keep-alive"}))
# The dummyserver responded with Connection:keep-alive, the connection
# persists.
conn = pool.pool.get()
self.assertNotEqual(conn.sock, None)
pool._put_conn(conn)
# Another request asking the server to close the connection. This one
# should get cleaned up for the next request.
r = yield From(pool.request("GET", "/keepalive?close=1", retries=0, headers={"Connection": "close"}))
self.assertEqual(r.status, 200)
conn = pool.pool.get()
self.assertEqual(conn.sock, None)
pool._put_conn(conn)
# Next request
r = yield From(pool.request("GET", "/keepalive?close=0"))
@asyncio.coroutine
def test_post_with_urlencode(self):
data = {"banana": "hammock", "lol": "cat"}
r = yield From(self.pool.request("POST", "/echo", fields=data, encode_multipart=False))
self.assertEqual((yield From(r.data).decode("utf-8"), urlencode(data)))
@asyncio.coroutine
def test_post_with_multipart(self):
data = {"banana": "hammock", "lol": "cat"}
r = yield From(self.pool.request("POST", "/echo", fields=data, encode_multipart=True))
body = (yield From(r.data).split(b"\r\n"))
encoded_data = encode_multipart_formdata(data)[0]
expected_body = encoded_data.split(b"\r\n")
# TODO: Get rid of extra parsing stuff when you can specify
# a custom boundary to encode_multipart_formdata
"""
We need to loop the return lines because a timestamp is attached
from within encode_multipart_formdata. When the server echos back
the data, it has the timestamp from when the data was encoded, which
is not equivalent to when we run encode_multipart_formdata on
the data again.
"""
for i, line in enumerate(body):
if line.startswith(b"--"):
continue
self.assertEqual(body[i], expected_body[i])
@asyncio.coroutine
def test_check_gzip(self):
r = yield From(self.pool.request("GET", "/encodingrequest", headers={"accept-encoding": "gzip"}))
self.assertEqual(r.headers.get("content-encoding"), "gzip")
self.assertEqual((yield From(r.data)), b"hello, world!")
@asyncio.coroutine
def test_check_deflate(self):
r = yield From(self.pool.request("GET", "/encodingrequest", headers={"accept-encoding": "deflate"}))
self.assertEqual(r.headers.get("content-encoding"), "deflate")
self.assertEqual((yield From(r.data)), b"hello, world!")
@asyncio.coroutine
def test_bad_decode(self):
yield From(
self.aioAssertRaises(
DecodeError,
self.pool.request,
"GET",
"/encodingrequest",
headers={"accept-encoding": "garbage-deflate"},
)
)
yield From(
self.aioAssertRaises(
DecodeError, self.pool.request, "GET", "/encodingrequest", headers={"accept-encoding": "garbage-gzip"}
)
)
@asyncio.coroutine
def test_connection_count(self):
pool = HTTPConnectionPool(self.host, self.port, maxsize=1)
yield From(pool.request("GET", "/"))
yield From(pool.request("GET", "/"))
yield From(pool.request("GET", "/"))
self.assertEqual(pool.num_connections, 1)
self.assertEqual(pool.num_requests, 3)
@asyncio.coroutine
def test_connection_count_bigpool(self):
http_pool = HTTPConnectionPool(self.host, self.port, maxsize=16)
yield From(http_pool.request("GET", "/"))
yield From(http_pool.request("GET", "/"))
yield From(http_pool.request("GET", "/"))
self.assertEqual(http_pool.num_connections, 1)
self.assertEqual(http_pool.num_requests, 3)
@asyncio.coroutine
def test_partial_response(self):
pool = HTTPConnectionPool(self.host, self.port, maxsize=1)
req_data = {"lol": "cat"}
resp_data = urlencode(req_data).encode("utf-8")
r = yield From(pool.request("GET", "/echo", fields=req_data, preload_content=False))
self.assertEqual(r.read(5), resp_data[:5])
self.assertEqual(r.read(), resp_data[5:])
@asyncio.coroutine
def test_lazy_load_twice(self):
# This test is sad and confusing. Need to figure out what's
# going on with partial reads and socket reuse.
pool = HTTPConnectionPool(self.host, self.port, block=True, maxsize=1, timeout=2)
payload_size = 1024 * 2
first_chunk = 512
boundary = "foo"
req_data = {"count": "a" * payload_size}
resp_data = encode_multipart_formdata(req_data, boundary=boundary)[0]
req2_data = {"count": "b" * payload_size}
resp2_data = encode_multipart_formdata(req2_data, boundary=boundary)[0]
r1 = yield From(
pool.request("POST", "/echo", fields=req_data, multipart_boundary=boundary, preload_content=False)
)
self.assertEqual(r1.read(first_chunk), resp_data[:first_chunk])
try:
r2 = yield From(
pool.request(
"POST",
"/echo",
fields=req2_data,
multipart_boundary=boundary,
preload_content=False,
pool_timeout=0.001,
)
)
# This branch should generally bail here, but maybe someday it will
# work? Perhaps by some sort of magic. Consider it a TODO.
self.assertEqual(r2.read(first_chunk), resp2_data[:first_chunk])
self.assertEqual((yield From(r1.read()), resp_data[first_chunk:]))
self.assertEqual((yield From(r2.read()), resp2_data[first_chunk:]))
self.assertEqual(pool.num_requests, 2)
except EmptyPoolError:
self.assertEqual((yield From(r1.read()), resp_data[first_chunk:]))
self.assertEqual(pool.num_requests, 1)
self.assertEqual(pool.num_connections, 1)
@asyncio.coroutine
def test_for_double_release(self):
MAXSIZE = 5
# Check default state
pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
self.assertEqual(pool.num_connections, 0)
self.assertEqual(pool.pool.qsize(), MAXSIZE)
# Make an empty slot for testing
pool.pool.get()
self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
# Check state after simple request
yield From(pool.urlopen("GET", "/"))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
# Check state without release
yield From(pool.urlopen("GET", "/", preload_content=False))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
yield From(pool.urlopen("GET", "/"))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
# Check state after read
yield From(pool.urlopen("GET", "/").data)
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
yield From(pool.urlopen("GET", "/"))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 2)
@asyncio.coroutine
def test_release_conn_parameter(self):
MAXSIZE = 5
pool = HTTPConnectionPool(self.host, self.port, maxsize=MAXSIZE)
self.assertEqual(pool.pool.qsize(), MAXSIZE)
# Make request without releasing connection
yield From(pool.request("GET", "/", release_conn=False, preload_content=False))
self.assertEqual(pool.pool.qsize(), MAXSIZE - 1)
@asyncio.coroutine
def test_dns_error(self):
pool = HTTPConnectionPool("thishostdoesnotexist.invalid", self.port, timeout=0.001)
yield From(self.aioAssertRaises(MaxRetryError, pool.request, "GET", "/test", retries=2))
@asyncio.coroutine
def test_source_address(self):
for addr in VALID_SOURCE_ADDRESSES:
pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False)
r = yield From(pool.request("GET", "/source_address"))
assert (
yield From(r.data) == b(addr[0]),
(
"expected the response to contain the source address {addr}, "
"but was {data}".format(data=(yield From(r.data)), addr=b(addr[0]))
),
)
@asyncio.coroutine
def test_source_address_error(self):
for addr in INVALID_SOURCE_ADDRESSES:
pool = HTTPConnectionPool(self.host, self.port, source_address=addr, retries=False)
yield From(self.aioAssertRaises(ProtocolError, pool.request, "GET", "/source_address"))
@asyncio.coroutine
def test_httplib_headers_case_insensitive(self):
HEADERS = {"Content-Length": "0", "Content-type": "text/plain", "Server": "TornadoServer/%s" % tornado.version}
r = yield From(self.pool.request("GET", "/specific_method", fields={"method": "GET"}))
self.assertEqual(HEADERS, dict(r.headers.items())) # to preserve case sensitivity
