def test_try_till_success(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=[IOError, IOError, None])
a = RetryWrapper(a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=0)
a.f()
self.assertEqual(a1.f.call_count, 3)
def test_failure_raises_if_all_tries_fail(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=[IOError, IOError])
a = RetryWrapper(a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=2)
with self.assertRaises(IOError):
a.f()
self.assertEqual(a1.f.call_count, 2)
def test_failure(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=[IOError, 1])
a = RetryWrapper(a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=2)
self.assertEqual(a.f(), 1)
self.assertEqual(a1.f.call_count, 2)
def test_success(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=None)
a = RetryWrapper(a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=2)
a.f()
a1.f.assert_called_once_with()
def test_try_till_success(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=[IOError, IOError, None])
a = RetryWrapper(
a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=0
)
a.f()
self.assertEqual(a1.f.call_count, 3)
def test_failure_raises_if_all_tries_fail(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=[IOError, IOError])
a = RetryWrapper(
a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=2
)
with self.assertRaises(IOError):
a.f()
self.assertEqual(a1.f.call_count, 2)
def test_failure(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=[IOError, 1])
a = RetryWrapper(
a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=2
)
self.assertEqual(a.f(), 1)
self.assertEqual(a1.f.call_count, 2)
def test_success(self):
a1 = Mock()
a1.f = Mock(__name__='f', side_effect=None)
a = RetryWrapper(
a1,
retry_if=lambda x: True,
backoff=0.0001,
max_tries=2
)
a.f()
a1.f.assert_called_once_with()
def _wrap_aws_client(raw_client, min_backoff=None):
"""Wrap a given boto3 Client object so that it can retry when
throttled."""
return RetryWrapper(raw_client,
retry_if=_is_retriable_client_error,
backoff=max(_AWS_BACKOFF, min_backoff or 0),
multiplier=_AWS_BACKOFF_MULTIPLIER,
max_tries=_AWS_MAX_TRIES)
def wrap_aws_conn(raw_conn):
"""Wrap a given boto Connection object so that it can retry when
throttled."""
def retry_if(ex):
"""Retry if we get a server error indicating throttling. Also
handle spurious 505s that are thought to be part of a load
balancer issue inside AWS."""
return ((isinstance(ex, boto.exception.BotoServerError) and
('Throttling' in ex.body or 'RequestExpired' in ex.body
or ex.status == 505))
or (isinstance(ex, socket.error)
and ex.args in ((104, 'Connection reset by peer'),
(110, 'Connection timed out'))))
return RetryWrapper(raw_conn,
retry_if=retry_if,
backoff=EMR_BACKOFF,
multiplier=EMR_BACKOFF_MULTIPLIER,
max_tries=EMR_MAX_TRIES)
