def test_retry_times(self):
"""
`retry_times` returns function that will retry given number of times
"""
can_retry = retry_times(3)
for exception in (DummyException(), NotImplementedError(),
ValueError()):
self.assertTrue(can_retry(Failure(exception)))
self.assertFalse(can_retry(Failure(DummyException())))
def test_transient_errors_except_terminates_on_provided_exceptions(self):
"""
If the failure is of a type provided to :func:`transient_errors_except`,
the function it returns will treat it as terminal (returns False)
"""
can_retry = transient_errors_except(DummyException)
self.assertFalse(can_retry(Failure(DummyException())))
def test_ignores_transient_failures_and_retries(self):
"""
Retries after interval if the ``do_work`` function errbacks with an
error that is ignored by the ``can_retry`` function. The error is
not propagated.
"""
wrapped_retry = mock.MagicMock(wraps=self.retry_function, spec=[])
d = retry(self.work_function, wrapped_retry, self.interval_function,
self.clock)
self.assertNoResult(d)
self.assertEqual(len(self.retries), 1)
# no result on errback
self.retries[-1].errback(DummyException('hey!'))
self.assertIsNone(self.successResultOf(self.retries[-1]))
self.assertNoResult(d)
wrapped_retry.assert_called_once_with(CheckFailure(DummyException))
self.clock.advance(self.interval)
# since it was an errback, loop retries the function again
self.assertNoResult(d)
self.assertEqual(len(self.retries), 2)
# stop loop
self.retries[-1].callback('result!')
self.assertEqual(self.successResultOf(d), 'result!')
def test_repeating_interval_always_returns_interval(self):
"""
``repeating_interval`` returns the same interval no matter what the
failure
"""
next_interval = repeating_interval(3)
for exception in (DummyException(), NotImplementedError()):
self.assertEqual(next_interval(Failure(exception)), 3)
def test_raise_error_on_code_does_not_match_code(self):
"""
``raise_error_on_code`` expects an APIError, and raises a particular
error given a specific code. Otherwise, it just wraps it in a
:class:`RequestError`
"""
failure = Failure(APIError(404, '', {}))
self.assertRaises(RequestError, raise_error_on_code,
failure, 500, DummyException(), 'url')
def test_pooled_deferred_errbbacks_not_obscured(self):
"""
The errbacks of pooled deferreds are not obscured by removing them
from the pool.
"""
holdup = Deferred()
self.pool.add(holdup)
holdup.errback(DummyException('hey'))
self.failureResultOf(holdup, DummyException)
def test_terminal_errors_except_defaults_to_all_errors_bad(self):
"""
If no args are provided to :func:`fail_unless`, the
function it returns treats all errors as terminal (returns False)
"""
can_retry = terminal_errors_except()
for exception in (DummyException(), NotImplementedError()):
self.assertFalse(can_retry(Failure(exception)))
def test_transient_errors_except_defaults_to_all_transient(self):
"""
If no args are provided to :func:`transient_errors_except`, the
function it returns treats all errors as transient (returns True)
"""
can_retry = transient_errors_except()
for exception in (DummyException(), NotImplementedError()):
self.assertTrue(can_retry(Failure(exception)))
def test_exp_backoff_interval(self):
"""
``exponential_backoff_interval`` returns previous interval * 2 every
time it is called
"""
err = DummyException()
next_interval = exponential_backoff_interval(3)
self.assertEqual(next_interval(err), 3)
self.assertEqual(next_interval(err), 6)
self.assertEqual(next_interval(err), 12)
def test_preserves_cancellation_function_errback(self):
"""
If a cancellation function that errbacks (with a non-CancelledError) is
provided to the deferred being cancelled, this other error will not be
converted to a TimedOutError.
"""
d = Deferred(lambda c: c.errback(DummyException('what!')))
timeout_deferred(d, 10, self.clock)
self.assertNoResult(d)
self.clock.advance(15)
self.failureResultOf(d, DummyException)
def test_retries_at_intervals_specified_by_interval_function(self):
"""
``do_work``, if it experiences transient failures, will be retried at
intervals returned by the ``next_interval`` function
"""
changing_interval = mock.MagicMock(spec=[])
d = retry(self.work_function, self.retry_function, changing_interval,
self.clock)
changing_interval.return_value = 1
self.assertEqual(len(self.retries), 1)
self.retries[-1].errback(DummyException('hey!'))
self.assertNoResult(d)
changing_interval.assert_called_once_with(CheckFailure(DummyException))
self.clock.advance(1)
changing_interval.return_value = 2
self.assertEqual(len(self.retries), 2)
self.retries[-1].errback(DummyException('hey!'))
self.assertNoResult(d)
changing_interval.assert_has_calls(
[mock.call(CheckFailure(DummyException))] * 2)
# the next interval has changed - after 1 second, it is still not
# retried
self.clock.advance(1)
self.assertEqual(len(self.retries), 2)
self.assertNoResult(d)
changing_interval.assert_has_calls(
[mock.call(CheckFailure(DummyException))] * 2)
# after 2 seconds, the function is retried
self.clock.advance(1)
self.assertEqual(len(self.retries), 3)
# stop retrying
self.retries[-1].callback('hey')
def test_propagates_failure_if_failed_before_timeout(self):
"""
The deferred errbacks with the failure if it fails before the
timeout (e.g. timing out the deferred does not obscure the errback
failure).
"""
clock = Clock()
d = Deferred()
timeout_deferred(d, 10, clock)
d.errback(DummyException("fail"))
self.failureResultOf(d, DummyException)
# the timeout never happens - no further errback occurs
clock.advance(15)
self.assertIsNone(self.successResultOf(d))
def test_verified_delete_does_not_propagate_verification_failure(self):
"""
verified_delete propagates deletions from server deletion
"""
clock = Clock()
self.treq.delete.return_value = succeed(
mock.Mock(spec=['code'], code=204))
self.treq.head.return_value = fail(DummyException('failure'))
self.treq.content.side_effect = lambda *args: succeed("")
d = verified_delete(self.log,
'http://url/',
'my-auth-token',
'serverId',
clock=clock)
self.assertIsNone(self.successResultOf(d))
def test_notify_does_not_notify_until_pooled_deferreds_errbacks(self):
"""
If there are one or more deferreds in the pool, ``notify_when_empty``
does not notify until they are fired - works with errbacks too.
"""
holdup = Deferred()
self.pool.add(holdup)
d = self.pool.notify_when_empty()
self.assertNoResult(d)
holdup.errback(DummyException('hey'))
self.successResultOf(d)
# don't leave unhandled Deferred lying around
self.failureResultOf(holdup)
def test_request_failure(self):
"""
On failed call to request, failure is returned and request logged
"""
d = logging_treq.request('patch', self.url, data='',
log=self.log, clock=self.clock)
self.treq.request.assert_called_once_with(
method='patch', url=self.url,
headers={'x-otter-request-id': ['uuid']}, data='')
self.assertNoResult(d)
self.clock.advance(5)
self.treq.request.return_value.errback(Failure(DummyException('e')))
self.failureResultOf(d, DummyException)
self._assert_failure_logging('patch', DummyException, 5)
def test_default_can_retry_function(self):
"""
If no ``can_retry`` function is provided, a default function treats
any failure as transient
"""
d = retry(self.work_function, None, self.interval_function, self.clock)
self.assertEqual(len(self.retries), 1)
self.retries[-1].errback(DummyException('temp'))
self.clock.advance(self.interval)
self.assertEqual(len(self.retries), 2)
self.retries[-1].errback(NotImplementedError())
self.assertNoResult(d)
def test_stops_on_non_transient_error(self):
"""
If ``do_work`` errbacks with something the ``can_retry`` function does
not ignore, the error is propagated up. ``do_work`` is not retried.
"""
d = retry(self.work_function, lambda *args: False,
self.interval_function, self.clock)
self.assertNoResult(d)
self.assertEqual(len(self.retries), 1)
self.retries[-1].errback(DummyException('fail!'))
self.failureResultOf(d, DummyException)
# work_function not called again
self.clock.advance(self.interval)
self.assertEqual(len(self.retries), 1)
def test_random_interval(self):
"""
``random_interval`` returns the different random interval each time it
is called
"""
next_interval = random_interval(5, 10)
intervals = set()
for exception in [
DummyException(),
NotImplementedError(),
ValueError(),
FloatingPointError(),
IOError()
]:
interval = next_interval(exception)
self.assertTrue(5 <= interval <= 10)
self.assertNotIn(interval, intervals)
intervals.add(interval)
def _test_method_failure(self, method):
"""
On failed call to ``method``, failure is returned and request logged
"""
request_function = getattr(logging_treq, method)
d = request_function(url=self.url, headers={}, data='', log=self.log,
clock=self.clock)
treq_function = getattr(self.treq, method)
treq_function.assert_called_once_with(
url=self.url, headers={'x-otter-request-id': ['uuid']}, data='')
self.assertNoResult(d)
self.clock.advance(5)
treq_function.return_value.errback(Failure(DummyException('e')))
self.failureResultOf(d, DummyException)
self._assert_failure_logging(method, DummyException, 5)
def test_cancelling_deferred_does_not_cancel_completed_work(self):
"""
Cancelling the deferred does not attempt to cancel previously
callbacked results from ``do_work``
"""
d = retry(self.work_function, self.retry_function,
self.interval_function, self.clock)
self.assertEqual(len(self.retries), 1)
self.retries[-1].errback(DummyException('temp'))
# cancel main deferred
d.cancel()
self.failureResultOf(d, CancelledError)
# work_function's deferred is not cancelled
self.assertEqual(self.retries[-1].cancel.call_count, 0)
self.assertIsNone(self.successResultOf(self.retries[-1]))
def test_default_next_interval_function(self):
"""
If no ``next_interval`` function is provided, a default function returns
5 no matter what the failure.
"""
d = retry(self.work_function, self.retry_function, None, self.clock)
self.assertEqual(len(self.retries), 1)
self.retries[-1].errback(DummyException('temp'))
self.clock.advance(5)
self.assertEqual(len(self.retries), 2)
self.retries[-1].errback(NotImplementedError())
self.clock.advance(5)
self.assertEqual(len(self.retries), 3)
self.assertNoResult(d)