def _on_retry_finished(self, g):
""" A callback method invoked when a retry finishes. Will decide whether it should be
attempted to retry the invocation again or give up notifying the uses via callback
service if retry limit is reached.
"""
# Was there any exception caught when retrying?
e = g.exception
if e:
# Can we retry again?
if e.remaining:
g = spawn_later(self.req_bunch.retry_seconds, self._retry,
e.remaining)
g.link(self._on_retry_finished)
# Reached the limit, warn users in logs, notify callback service and give up.
else:
msg = retry_limit_reached_msg(self.req_bunch.retry_repeats,
self.req_bunch.target,
self.req_bunch.retry_seconds,
self.req_bunch.orig_cid)
self.logger.warn(msg)
self._notify_callback(False, None)
# Let the callback know it's all good
else:
self._notify_callback(True, g.value)
def test_invoke_retry_exception_no_async(self):
class Sleep(object):
def __init__(self):
self.times_called = 0
self.retry_seconds = []
def __call__(self, retry_seconds):
self.times_called += 1
self.retry_seconds.append(retry_seconds)
sleep = Sleep()
with patch('zato.server.pattern.invoke_retry.sleep', sleep):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback,
callback_impl_name,
cid,
expected_result,
raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': False,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke(target, 1, 2, 3, **kwargs)
except ZatoException as e:
expected_msg = retry_limit_reached_msg(kwargs_copy['repeats'],
target,
kwargs_copy['seconds'],
invoking_service.cid)
self.assertEquals(e.cid, cid)
self.assertEquals(e.message, expected_msg)
self.assertEquals(invoking_service.invoke_called_times,
kwargs_copy['repeats'])
self.assertEquals(sleep.times_called,
kwargs_copy['repeats'] - 1)
self.assertEquals(sleep.retry_seconds,
[kwargs_copy['seconds']] *
(kwargs_copy['repeats'] - 1))
else:
self.fail('Expected a ZatoException')
def test_retry_limit_reached_msg(self):
retry_repeats = rand_int()
service_name = rand_string()
retry_seconds = rand_int()
orig_cid = rand_string()
msg = retry_limit_reached_msg(retry_repeats, service_name, retry_seconds, orig_cid)
self.assertEquals(msg, '({}/{}) Retry limit reached for:`{}`, retry_seconds:`{}`, orig_cid:`{}`'.format(
retry_repeats, retry_repeats, service_name, retry_seconds, orig_cid))
def test_retry_limit_reached_msg(self):
retry_repeats = rand_int()
service_name = rand_string()
retry_seconds = rand_int()
orig_cid = rand_string()
msg = retry_limit_reached_msg(retry_repeats, service_name,
retry_seconds, orig_cid)
self.assertEquals(
msg,
'({}/{}) Retry limit reached for:`{}`, retry_seconds:`{}`, orig_cid:`{}`'
.format(retry_repeats, retry_repeats, service_name, retry_seconds,
orig_cid))
def test_invoke_retry_exception_no_async(self):
class Sleep(object):
def __init__(self):
self.times_called = 0
self.retry_seconds = []
def __call__(self, retry_seconds):
self.times_called += 1
self.retry_seconds.append(retry_seconds)
sleep = Sleep()
with patch('zato.server.pattern.invoke_retry.sleep', sleep):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback, callback_impl_name, cid, expected_result, raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': False,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke(target, 1, 2, 3, **kwargs)
except ZatoException, e:
expected_msg = retry_limit_reached_msg(kwargs_copy['repeats'], target, kwargs_copy['seconds'], invoking_service.cid)
self.assertEquals(e.cid, cid)
self.assertEquals(e.message, expected_msg)
self.assertEquals(invoking_service.invoke_called_times, kwargs_copy['repeats'])
self.assertEquals(sleep.times_called, kwargs_copy['repeats']-1)
self.assertEquals(sleep.retry_seconds, [kwargs_copy['seconds']] * (kwargs_copy['repeats']-1))
else:
