def test_retry_higher_total_loses_vs_read(self):
""" A lower read timeout than the total is honored """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
def test_string(self):
""" Retry string representation looks the way we expect """
retry = Retry()
self.assertEqual(str(retry), 'Retry(total=10, connect=None, read=None, redirect=None)')
for _ in range(3):
retry = retry.increment()
self.assertEqual(str(retry), 'Retry(total=7, connect=None, read=None, redirect=None)')
def test_retry_higher_total_loses(self):
""" A lower connect timeout than the total is honored """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
def test_retry_higher_total_loses_vs_read(self):
""" A lower read timeout than the total is honored """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
def test_retry_higher_total_loses(self):
""" A lower connect timeout than the total is honored """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=3)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
def test_retry_read_zero(self):
""" No second chances on read timeouts, by default """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=0)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
def test_retry_read_zero(self):
""" No second chances on read timeouts, by default """
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(read=0)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
def test_string(self):
""" Retry string representation looks the way we expect """
retry = Retry()
self.assertEqual(
str(retry),
'Retry(total=10, connect=None, read=None, redirect=None)')
for _ in range(3):
retry = retry.increment()
self.assertEqual(
str(retry),
'Retry(total=7, connect=None, read=None, redirect=None)')
def test_retry_both_specified(self):
"""Total can win if it's lower than the connect value"""
error = ConnectTimeoutError()
retry = Retry(connect=3, total=2)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
def test_retry_both_specified(self):
"""Total can win if it's lower than the connect value"""
error = ConnectTimeoutError()
retry = Retry(connect=3, total=2)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
def test_read_retries(self):
""" Should retry for status codes in the whitelist """
retry = Retry(read=1, status_forcelist=[418])
resp = yield From(self.pool.request('GET', '/successful_retry',
headers={'test-name': 'test_read_retries'},
retries=retry))
self.assertEqual(resp.status, 200)
def test_read_total_retries(self):
""" HTTP response w/ status code in the whitelist should be retried """
headers = {'test-name': 'test_read_total_retries'}
retry = Retry(total=1, status_forcelist=[418])
resp = yield From(self.pool.request('GET', '/successful_retry',
headers=headers, retries=retry))
self.assertEqual(resp.status, 200)
def test_retries_wrong_whitelist(self):
"""HTTP response w/ status code not in whitelist shouldn't be retried"""
retry = Retry(total=1, status_forcelist=[202])
resp = yield From(self.pool.request('GET', '/successful_retry',
headers={'test-name': 'test_wrong_whitelist'},
retries=retry))
self.assertEqual(resp.status, 418)
def test_connect_timeout(self):
url = '/sleep?seconds=0.005'
timeout = Timeout(connect=0.001)
# Pool-global timeout
pool = HTTPConnectionPool(TARPIT_HOST, self.port, timeout=timeout)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET', url))
# Retries
retries = Retry(connect=0)
yield From(self.aioAssertRaises(MaxRetryError, pool.request, 'GET', url,
retries=retries))
# Request-specific connection timeouts
big_timeout = Timeout(read=0.2, connect=0.2)
pool = HTTPConnectionPool(TARPIT_HOST, self.port,
timeout=big_timeout, retries=False)
conn = pool._get_conn()
yield From(self.aioAssertRaises(ConnectTimeoutError, pool._make_request, conn, 'GET',
url, timeout=timeout))
pool._put_conn(conn)
yield From(self.aioAssertRaises(ConnectTimeoutError, pool.request, 'GET', url,
timeout=timeout))
def test_default_method_whitelist_retried(self):
""" urllib3 should retry methods in the default method whitelist """
retry = Retry(total=1, status_forcelist=[418])
resp = yield From(self.pool.request('OPTIONS', '/successful_retry',
headers={'test-name': 'test_default_whitelist'},
retries=retry))
self.assertEqual(resp.status, 200)
def test_status_forcelist(self):
retry = Retry(status_forcelist=xrange(500, 600))
self.assertFalse(retry.is_forced_retry('GET', status_code=200))
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=500))
retry = Retry(total=1, status_forcelist=[418])
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=418))
def test_retries_wrong_method_list(self):
"""Method not in our whitelist should not be retried, even if code matches"""
headers = {'test-name': 'test_wrong_method_whitelist'}
retry = Retry(total=1, status_forcelist=[418],
method_whitelist=['POST'])
resp = yield From(self.pool.request('GET', '/successful_retry',
headers=headers, retries=retry))
self.assertEqual(resp.status, 418)
def test_retry_total_none(self):
""" if Total is none, connect error should take precedence """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertFalse(retry.is_exhausted())
def test_retry_reuse_safe(self):
""" It should be possible to reuse a Retry object across requests """
headers = {'test-name': 'test_retry_safe'}
retry = Retry(total=1, status_forcelist=[418])
resp = yield From(self.pool.request('GET', '/successful_retry',
headers=headers, retries=retry))
self.assertEqual(resp.status, 200)
resp = yield From(self.pool.request('GET', '/successful_retry',
headers=headers, retries=retry))
self.assertEqual(resp.status, 200)
def test_connection_error_retries(self):
""" ECONNREFUSED error should raise a connection error, with retries """
port = find_unused_port()
pool = HTTPConnectionPool(self.host, port)
try:
yield From(pool.request('GET', '/', retries=Retry(connect=3)))
self.fail("Should have failed with a connection error.")
except MaxRetryError as e:
self.assertTrue(isinstance(e.reason, ProtocolError))
self.assertEqual(e.reason.args[1].errno, errno.ECONNREFUSED)
def test_disabled_retry(self):
""" Disabled retries should disable redirect handling. """
r = yield From(self.pool.request('GET', '/redirect',
fields={'target': '/'},
retries=False))
self.assertEqual(r.status, 303)
r = yield From(self.pool.request('GET', '/redirect',
fields={'target': '/'},
retries=Retry(redirect=False)))
self.assertEqual(r.status, 303)
def test_status_forcelist(self):
retry = Retry(status_forcelist=xrange(500,600))
self.assertFalse(retry.is_forced_retry('GET', status_code=200))
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=500))
retry = Retry(total=1, status_forcelist=[418])
self.assertFalse(retry.is_forced_retry('GET', status_code=400))
self.assertTrue(retry.is_forced_retry('GET', status_code=418))
def test_retry_total_none(self):
""" if Total is none, connect error should take precedence """
error = ConnectTimeoutError()
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
try:
retry.increment(error=error)
self.fail("Failed to raise error.")
except MaxRetryError as e:
self.assertEqual(e.reason, error)
error = ReadTimeoutError(None, "/", "read timed out")
retry = Retry(connect=2, total=None)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
retry = retry.increment(error=error)
self.assertFalse(retry.is_exhausted())
def test_retry_weird_http_version(self):
""" Retry class should handle httplib.BadStatusLine errors properly """
def socket_handler(listener):
sock = listener.accept()[0]
# First request.
# Pause before responding so the first request times out.
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# send unknown http protocol
body = "bad http 0.5 response"
sock.send(('HTTP/0.5 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'%s' % (len(body), body)).encode('utf-8'))
sock.close()
# Second request.
sock = listener.accept()[0]
buf = b''
while not buf.endswith(b'\r\n\r\n'):
buf += sock.recv(65536)
# Now respond immediately.
sock.send(('HTTP/1.1 200 OK\r\n'
'Content-Type: text/plain\r\n'
'Content-Length: %d\r\n'
'\r\n'
'foo' % (len('foo'))).encode('utf-8'))
sock.close() # Close the socket.
self._start_server(socket_handler)
pool = HTTPConnectionPool(self.host, self.port)
retry = Retry(read=1)
response = yield From(pool.request('GET', '/', retries=retry))
self.assertEqual(response.status, 200)
self.assertEqual((yield From(response.data), b'foo'))
def test_retry_default(self):
""" If no value is specified, should retry connects 3 times """
retry = Retry()
self.assertEqual(retry.total, 10)
self.assertEqual(retry.connect, None)
self.assertEqual(retry.read, None)
self.assertEqual(retry.redirect, None)
error = ConnectTimeoutError()
retry = Retry(connect=1)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
retry = Retry(connect=1)
retry = retry.increment(error=error)
self.assertFalse(retry.is_exhausted())
self.assertTrue(Retry(0).raise_on_redirect)
self.assertFalse(Retry(False).raise_on_redirect)
def test_exhausted(self):
self.assertFalse(Retry(0).is_exhausted())
self.assertTrue(Retry(-1).is_exhausted())
self.assertEqual(Retry(1).increment().total, 0)
def test_sleep(self):
# sleep a very small amount of time so our code coverage is happy
retry = Retry(backoff_factor=0.0001)
retry = retry.increment()
retry = retry.increment()
retry.sleep()
def test_backoff(self):
""" Backoff is computed correctly """
max_backoff = Retry.BACKOFF_MAX
retry = Retry(total=100, backoff_factor=0.2)
self.assertEqual(retry.get_backoff_time(), 0) # First request
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0) # First retry
retry = retry.increment()
self.assertEqual(retry.backoff_factor, 0.2)
self.assertEqual(retry.total, 98)
self.assertEqual(retry.get_backoff_time(), 0.4) # Start backoff
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0.8)
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 1.6)
for i in xrange(10):
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), max_backoff)
def test_retry_default(self):
""" If no value is specified, should retry connects 3 times """
retry = Retry()
self.assertEqual(retry.total, 10)
self.assertEqual(retry.connect, None)
self.assertEqual(retry.read, None)
self.assertEqual(retry.redirect, None)
error = ConnectTimeoutError()
retry = Retry(connect=1)
retry = retry.increment(error=error)
self.assertRaises(MaxRetryError, retry.increment, error=error)
retry = Retry(connect=1)
retry = retry.increment(error=error)
self.assertFalse(retry.is_exhausted())
self.assertTrue(Retry(0).raise_on_redirect)
self.assertFalse(Retry(False).raise_on_redirect)
def test_zero_backoff(self):
retry = Retry()
self.assertEqual(retry.get_backoff_time(), 0)
retry = retry.increment()
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0)
def test_sleep(self):
# sleep a very small amount of time so our code coverage is happy
retry = Retry(backoff_factor=0.0001)
retry = retry.increment()
retry = retry.increment()
retry.sleep()
def test_zero_backoff(self):
retry = Retry()
self.assertEqual(retry.get_backoff_time(), 0)
retry = retry.increment()
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0)
def test_disabled(self):
self.assertRaises(MaxRetryError, Retry(-1).increment)
self.assertRaises(MaxRetryError, Retry(0).increment)
def test_backoff(self):
""" Backoff is computed correctly """
max_backoff = Retry.BACKOFF_MAX
retry = Retry(total=100, backoff_factor=0.2)
self.assertEqual(retry.get_backoff_time(), 0) # First request
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0) # First retry
retry = retry.increment()
self.assertEqual(retry.backoff_factor, 0.2)
self.assertEqual(retry.total, 98)
self.assertEqual(retry.get_backoff_time(), 0.4) # Start backoff
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 0.8)
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), 1.6)
for i in xrange(10):
retry = retry.increment()
self.assertEqual(retry.get_backoff_time(), max_backoff)