def testSucceedLater(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(SucceedLater(1)) decorator = retry(try_max=9999, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) result = loop.run_until_complete(decorated_func()) self.assertEqual(result, 'success')
def testSucceedNever(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, SucceedNeverException))
def testHangForeverReraise(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(reraise=True, try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception(), asyncio.TimeoutError))
def testSucceedLater(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(SucceedLater(1))
decorator = retry(try_max=9999,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
result = loop.run_until_complete(decorated_func())
self.assertEqual(result, 'success')
def testSucceedLater(self): loop = global_event_loop() func = SucceedLater(1) func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_max=9999, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) result = loop.run_until_complete(decorated_func()) self.assertEqual(result, 'success')
def testHangForever(self): loop = global_event_loop() func = HangForever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_max=2, try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) done, pending = loop.run_until_complete(wait([decorated_func()])) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done[0].exception().__cause__, TimeoutError))
def testSucceedNeverReraise(self): loop = global_event_loop() func = SucceedNever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(reraise=True, try_max=4, try_timeout=None, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) done, pending = loop.run_until_complete(wait([decorated_func()])) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done[0].exception(), SucceedNeverException))
def testSucceedLater(self):
loop = global_event_loop()._asyncio_wrapper
func = SucceedLater(1)
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_max=9999,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
result = loop.run_until_complete(decorated_func())
self.assertEqual(result, 'success')
def testOverallTimeoutWithTimeoutError(self): loop = global_event_loop() # results in TimeoutError because it hangs forever func_coroutine = self._wrap_coroutine_func(HangForever()) decorator = retry(try_timeout=0.1, overall_timeout=0.3, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) done, pending = loop.run_until_complete(asyncio.wait([decorated_func()], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
def testCancelRetry(self): loop = global_event_loop() func_coroutine = self._wrap_coroutine_func(SucceedNever()) decorator = retry(try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine, loop=loop) future = decorated_func() loop.call_later(0.3, future.cancel) done, pending = loop.run_until_complete(asyncio.wait([future], loop=loop)) self.assertEqual(len(done), 1) self.assertTrue(done.pop().cancelled())
def testCancelRetry(self): loop = global_event_loop() func = SucceedNever() func_coroutine = functools.partial(loop.run_in_executor, None, func) decorator = retry(try_timeout=0.1, delay_func=RandomExponentialBackoff(multiplier=0.1, base=2)) decorated_func = decorator(func_coroutine) future = decorated_func() loop.call_later(0.3, future.cancel) done, pending = loop.run_until_complete(wait([future])) self.assertEqual(len(done), 1) self.assertTrue(done[0].cancelled())
def testHangForever(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(HangForever())
decorator = retry(try_max=2,
try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
def testSucceedNeverReraise(self):
loop = global_event_loop()
func = SucceedNever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(reraise=True,
try_max=4,
try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(wait([decorated_func()]))
self.assertEqual(len(done), 1)
self.assertTrue(isinstance(done[0].exception(), SucceedNeverException))
def testHangForever(self):
loop = global_event_loop()
func = HangForever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_max=2,
try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(wait([decorated_func()]))
self.assertEqual(len(done), 1)
self.assertTrue(isinstance(done[0].exception().__cause__,
TimeoutError))
def testCancelRetry(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
future = decorated_func()
loop.call_later(0.3, future.cancel)
done, pending = loop.run_until_complete(
asyncio.wait([future], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(done.pop().cancelled())
def testCancelRetry(self):
loop = global_event_loop()
func = SucceedNever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
future = decorated_func()
loop.call_later(0.3, future.cancel)
done, pending = loop.run_until_complete(wait([future]))
self.assertEqual(len(done), 1)
self.assertTrue(done[0].cancelled())
def testSucceedNeverReraise(self):
loop = global_event_loop()
with self._wrap_coroutine_func(SucceedNever()) as func_coroutine:
decorator = retry(reraise=True,
try_max=4,
try_timeout=None,
delay_func=RandomExponentialBackoff(
multiplier=0.1, base=2))
decorated_func = decorator(func_coroutine, loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception(), SucceedNeverException))
def testOverallTimeoutWithException(self):
loop = global_event_loop()
func_coroutine = self._wrap_coroutine_func(SucceedNever())
decorator = retry(try_timeout=0.1,
overall_timeout=0.3,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine, loop=loop)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__,
SucceedNeverException))
def testSucceedNever(self):
loop = global_event_loop()._asyncio_wrapper
func = SucceedNever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_max=4,
try_timeout=None,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__,
SucceedNeverException))
def testOverallTimeoutWithTimeoutError(self):
loop = global_event_loop()._asyncio_wrapper
# results in TimeoutError because it hangs forever
func = HangForever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(try_timeout=0.1,
overall_timeout=0.3,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception().__cause__, asyncio.TimeoutError))
def testHangForeverReraise(self):
loop = global_event_loop()._asyncio_wrapper
func = HangForever()
func_coroutine = functools.partial(loop.run_in_executor, None, func)
decorator = retry(reraise=True,
try_max=2,
try_timeout=0.1,
delay_func=RandomExponentialBackoff(multiplier=0.1,
base=2))
decorated_func = decorator(func_coroutine)
done, pending = loop.run_until_complete(
asyncio.wait([decorated_func()], loop=loop))
self.assertEqual(len(done), 1)
self.assertTrue(
isinstance(done.pop().exception(), asyncio.TimeoutError))
def _key_refresh_retry_decorator(self):
"""
Return a retry decorator, or None if retry is disabled.
If retry fails, the function reraises the exception raised
by the decorated function. If retry times out and no exception
is available to reraise, the function raises TimeoutError.
"""
errors = []
if self.repo.sync_openpgp_key_refresh_retry_count is None:
return None
try:
retry_count = int(self.repo.sync_openpgp_key_refresh_retry_count)
except Exception as e:
errors.append("sync-openpgp-key-refresh-retry-count: {}".format(e))
else:
if retry_count <= 0:
return None
if self.repo.sync_openpgp_key_refresh_retry_overall_timeout is None:
retry_overall_timeout = None
else:
try:
retry_overall_timeout = float(
self.repo.sync_openpgp_key_refresh_retry_overall_timeout)
except Exception as e:
errors.append(
"sync-openpgp-key-refresh-retry-overall-timeout: {}".
format(e))
else:
if retry_overall_timeout < 0:
errors.append(
"sync-openpgp-key-refresh-retry-overall-timeout: "
"value must be greater than or equal to zero: {}".
format(retry_overall_timeout))
elif retry_overall_timeout == 0:
retry_overall_timeout = None
if self.repo.sync_openpgp_key_refresh_retry_delay_mult is None:
retry_delay_mult = None
else:
try:
retry_delay_mult = float(
self.repo.sync_openpgp_key_refresh_retry_delay_mult)
except Exception as e:
errors.append(
"sync-openpgp-key-refresh-retry-delay-mult: {}".format(e))
else:
if retry_delay_mult <= 0:
errors.append("sync-openpgp-key-refresh-retry-mult: "
"value must be greater than zero: {}".format(
retry_delay_mult))
if self.repo.sync_openpgp_key_refresh_retry_delay_exp_base is None:
retry_delay_exp_base = None
else:
try:
retry_delay_exp_base = float(
self.repo.sync_openpgp_key_refresh_retry_delay_exp_base)
except Exception as e:
errors.append(
"sync-openpgp-key-refresh-retry-delay-exp: {}".format(e))
else:
if retry_delay_exp_base <= 0:
errors.append("sync-openpgp-key-refresh-retry-delay-exp: "
"value must be greater than zero: {}".format(
retry_delay_mult))
if errors:
lines = []
lines.append("")
lines.append("!!! Retry disabled for openpgp key refresh:")
lines.append("")
for msg in errors:
lines.append(" {}".format(msg))
lines.append("")
for line in lines:
writemsg_level("{}\n".format(line),
level=logging.ERROR,
noiselevel=-1)
return None
return retry(
reraise=True,
try_max=retry_count,
overall_timeout=(retry_overall_timeout
if retry_overall_timeout > 0 else None),
delay_func=RandomExponentialBackoff(
multiplier=(1
if retry_delay_mult is None else retry_delay_mult),
base=(2 if retry_delay_exp_base is None else
retry_delay_exp_base),
),
)
def _key_refresh_retry_decorator(self):
'''
Return a retry decorator, or None if retry is disabled.
If retry fails, the function reraises the exception raised
by the decorated function. If retry times out and no exception
is available to reraise, the function raises TimeoutError.
'''
errors = []
if self.repo.sync_openpgp_key_refresh_retry_count is None:
return None
try:
retry_count = int(self.repo.sync_openpgp_key_refresh_retry_count)
except Exception as e:
errors.append('sync-openpgp-key-refresh-retry-count: {}'.format(e))
else:
if retry_count <= 0:
return None
if self.repo.sync_openpgp_key_refresh_retry_overall_timeout is None:
retry_overall_timeout = None
else:
try:
retry_overall_timeout = float(self.repo.sync_openpgp_key_refresh_retry_overall_timeout)
except Exception as e:
errors.append('sync-openpgp-key-refresh-retry-overall-timeout: {}'.format(e))
else:
if retry_overall_timeout < 0:
errors.append('sync-openpgp-key-refresh-retry-overall-timeout: '
'value must be greater than or equal to zero: {}'.format(retry_overall_timeout))
elif retry_overall_timeout == 0:
retry_overall_timeout = None
if self.repo.sync_openpgp_key_refresh_retry_delay_mult is None:
retry_delay_mult = None
else:
try:
retry_delay_mult = float(self.repo.sync_openpgp_key_refresh_retry_delay_mult)
except Exception as e:
errors.append('sync-openpgp-key-refresh-retry-delay-mult: {}'.format(e))
else:
if retry_delay_mult <= 0:
errors.append('sync-openpgp-key-refresh-retry-mult: '
'value must be greater than zero: {}'.format(retry_delay_mult))
if self.repo.sync_openpgp_key_refresh_retry_delay_exp_base is None:
retry_delay_exp_base = None
else:
try:
retry_delay_exp_base = float(self.repo.sync_openpgp_key_refresh_retry_delay_exp_base)
except Exception as e:
errors.append('sync-openpgp-key-refresh-retry-delay-exp: {}'.format(e))
else:
if retry_delay_exp_base <= 0:
errors.append('sync-openpgp-key-refresh-retry-delay-exp: '
'value must be greater than zero: {}'.format(retry_delay_mult))
if errors:
lines = []
lines.append('')
lines.append('!!! Retry disabled for openpgp key refresh:')
lines.append('')
for msg in errors:
lines.append(' {}'.format(msg))
lines.append('')
for line in lines:
writemsg_level("{}\n".format(line),
level=logging.ERROR, noiselevel=-1)
return None
return retry(
reraise=True,
try_max=retry_count,
overall_timeout=(retry_overall_timeout if retry_overall_timeout > 0 else None),
delay_func=RandomExponentialBackoff(
multiplier=(1 if retry_delay_mult is None else retry_delay_mult),
base=(2 if retry_delay_exp_base is None else retry_delay_exp_base)))
