def test_invoke_retry_ok(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyInvokingService(callback, callback_impl_name,
cid, expected_result)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
}
result = ir.invoke_retry(target, 1, 2, 3, **kwargs)
self.assertEquals(expected_result, result)
self.assertTrue(len(invoking_service.invoke_args), 2)
self.assertEquals(invoking_service.invoke_args, (target, 1, 2, 3))
self.assertEquals(invoking_service.invoke_kwargs, {})
def test_invoke_retry_exception_has_async(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyInvokingService(callback, callback_impl_name, cid, expected_result, raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke_retry(target, 1, 2, 3, **kwargs)
except NeedsRetry, e:
self.assertEquals(e.cid, cid)
self.assertEquals(e.cid, e.inner_exc.message)
def test_get_retry_settings_has_invalid_async_callback(self):
ir = InvokeRetry(None)
callback = [None, rand_string()]
repeats = [None, rand_int()]
target = rand_string()
for callback_item in callback:
for repeats_item in repeats:
kwargs = {
'async_fallback': True,
'callback': callback_item,
'repeats': repeats_item,
}
try:
ir._get_retry_settings(target, **kwargs)
except ValueError, e:
for name in 'callback', 'repeats':
if name in e.message:
self.assertEquals(e.message, 'Could not invoke `{}`, `{}` was not provided ({})'.format(
target, name, None))
else:
self.assertTrue(callback_item is not None)
self.assertTrue(repeats_item is not None)
def test_get_retry_settings_has_invalid_async_callback(self):
ir = InvokeRetry(None)
callback = [None, rand_string()]
repeats = [None, rand_int()]
target = rand_string()
for callback_item in callback:
for repeats_item in repeats:
kwargs = {
'async_fallback': True,
'callback': callback_item,
'repeats': repeats_item,
}
try:
ir._get_retry_settings(target, **kwargs)
except ValueError, e:
for name in 'callback', 'repeats':
if name in e.message:
self.assertEquals(
e.message,
'Could not invoke `{}`, `{}` was not provided ({})'
.format(target, name, None))
else:
self.assertTrue(callback_item is not None)
self.assertTrue(repeats_item is not None)
def test_invoke_retry_ok(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback, callback_impl_name, cid, expected_result)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
'cid': cid,
}
result = ir.invoke(target, 1, 2, 3, **kwargs)
self.assertEquals(expected_result, result)
self.assertTrue(len(invoking_service.invoke_args), 2)
self.assertEquals(invoking_service.invoke_args, (target, 1, 2, 3))
self.assertEquals(invoking_service.invoke_kwargs, {'cid':cid})
def test_invoke_retry_exception_no_async(self):
class Sleep(object):
def __init__(self):
self.times_called = 0
self.retry_seconds = []
def __call__(self, retry_seconds):
self.times_called += 1
self.retry_seconds.append(retry_seconds)
sleep = Sleep()
with patch('zato.server.pattern.invoke_retry.sleep', sleep):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback,
callback_impl_name,
cid,
expected_result,
raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': False,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke(target, 1, 2, 3, **kwargs)
except ZatoException as e:
expected_msg = retry_limit_reached_msg(kwargs_copy['repeats'],
target,
kwargs_copy['seconds'],
invoking_service.cid)
self.assertEquals(e.cid, cid)
self.assertEquals(e.message, expected_msg)
self.assertEquals(invoking_service.invoke_called_times,
kwargs_copy['repeats'])
self.assertEquals(sleep.times_called,
kwargs_copy['repeats'] - 1)
self.assertEquals(sleep.retry_seconds,
[kwargs_copy['seconds']] *
(kwargs_copy['repeats'] - 1))
else:
self.fail('Expected a ZatoException')
def test_invoke_retry_exception_has_async(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyInvokingService(callback,
callback_impl_name,
cid,
expected_result,
raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
try:
ir.invoke_retry(target, 1, 2, 3, **kwargs)
except NeedsRetry, e:
self.assertEquals(e.cid, cid)
self.assertEquals(e.cid, e.inner_exc.message)
def test_get_retry_settings_has_async_callback(self):
callback = rand_string()
callback_impl_name = rand_string()
invoking_service = DummyInvokingService(callback, callback_impl_name)
for needs_seconds in True, False:
target = rand_string()
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {
rand_string(): rand_string(),
rand_int(): rand_int()
},
'repeats': rand_int(),
'seconds': rand_int() if needs_seconds else 0,
'minutes': rand_int() if not needs_seconds else 0,
'custom1': 'custom1',
'custom2': 'custom2',
}
kwargs_copy = deepcopy(kwargs)
ir = InvokeRetry(invoking_service)
async_fallback, callback, callback_context, retry_repeats, retry_seconds, kwargs = ir._get_retry_settings(
target, **kwargs)
self.assertTrue(async_fallback)
self.assertNotIn('callback', kwargs)
self.assertEquals(callback, kwargs_copy['callback'])
self.assertNotIn('context', kwargs)
self.assertDictEqual(callback_context, kwargs_copy['context'])
self.assertNotIn('repeats', kwargs)
self.assertEquals(retry_repeats, kwargs_copy['repeats'])
if needs_seconds:
self.assertNotIn('seconds', kwargs)
self.assertEquals(retry_seconds, kwargs_copy['seconds'])
else:
self.assertNotIn('minutes', kwargs)
self.assertEquals(retry_seconds, kwargs_copy['minutes'] * 60)
self.assertDictEqual(kwargs, {
'custom1': 'custom1',
'custom2': 'custom2'
})
def test_invoke_retry_exception_no_async(self):
class Sleep(object):
def __init__(self):
self.times_called = 0
self.retry_seconds = []
def __call__(self, retry_seconds):
self.times_called += 1
self.retry_seconds.append(retry_seconds)
sleep = Sleep()
with patch('zato.server.pattern.invoke_retry.sleep', sleep):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
expected_result = rand_string()
invoking_service = DummyTargetService(callback, callback_impl_name, cid, expected_result, raise_on_invoke=True)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': False,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(1, 10),
'seconds': 0.01,
'minutes': 0,
}
kwargs_copy = deepcopy(kwargs)
try:
ir.invoke(target, 1, 2, 3, **kwargs)
except ZatoException, e:
expected_msg = retry_limit_reached_msg(kwargs_copy['repeats'], target, kwargs_copy['seconds'], invoking_service.cid)
self.assertEquals(e.cid, cid)
self.assertEquals(e.message, expected_msg)
self.assertEquals(invoking_service.invoke_called_times, kwargs_copy['repeats'])
self.assertEquals(sleep.times_called, kwargs_copy['repeats']-1)
self.assertEquals(sleep.retry_seconds, [kwargs_copy['seconds']] * (kwargs_copy['repeats']-1))
else:
def test_invoke_async_retry(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
invoking_service = DummyTargetService(callback, callback_impl_name,
cid)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {
rand_string(): rand_string()
},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
'cid': cid
}
kwargs_copy = deepcopy(kwargs)
call_cid = ir.invoke_async(target, **kwargs)
self.assertTrue(len(invoking_service.invoke_async_args), 2)
self.assertEquals(invoking_service.invoke_async_kwargs, {'cid': cid})
invoke_retry_service, retry_request = invoking_service.invoke_async_args
self.assertEquals(invoke_retry_service,
'zato.pattern.invoke-retry.invoke-retry')
retry_request = loads(retry_request)
self.assertEquals(retry_request['target'], target)
self.assertEquals(retry_request['callback'], callback)
self.assertEquals(retry_request['args'], [])
self.assertEquals(retry_request['retry_seconds'],
kwargs_copy['seconds'])
self.assertEquals(retry_request['kwargs'], {'cid': call_cid})
self.assertEquals(retry_request['orig_cid'], cid)
self.assertEquals(retry_request['retry_repeats'],
kwargs_copy['repeats'])
self.assertEquals(retry_request['callback_context'],
kwargs_copy['context'])
def test_get_retry_settings_has_async_callback_both_secs_mins(self):
ir = InvokeRetry(None)
target = rand_string()
kwargs = {
'async_fallback': True,
'callback': rand_string(),
'context': {},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': rand_int(),
}
try:
ir._get_retry_settings(target, **kwargs)
except ValueError, e:
self.assertEquals(e.message, 'Could not invoke `{}`, only one of seconds:`{}` and minutes:`{}` can be given'.format(
target, kwargs['seconds'], kwargs['minutes']))
def test_get_retry_settings_has_async_callback_invalid_target(self):
ir = InvokeRetry(DummyTargetService())
kwargs = {
'async_fallback': True,
'callback': rand_string(),
'context': {},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
}
try:
ir._get_retry_settings(rand_string(), **kwargs)
except ValueError, e:
prefix = 'Service:`{}` does not exist, e:`Traceback (most recent call last):'.format(kwargs['callback'])
self.assertTrue(e.message.startswith(prefix))
self.assertIn("KeyError: u'{}'".format(kwargs['callback']), e.message)
def test_get_retry_settings_has_async_callback_no_secs_mins(self):
ir = InvokeRetry(None)
target = rand_string()
kwargs = {
'async_fallback': True,
'callback': rand_string(),
'context': {},
'repeats': rand_int(),
'seconds': 0,
'minutes': 0,
}
try:
ir._get_retry_settings(target, **kwargs)
except ValueError, e:
self.assertEquals(
e.message,
'Could not invoke `{}`, exactly one of seconds:`{}` or minutes:`{}` must be given'
.format(target, kwargs['seconds'], kwargs['minutes']))
def test_get_retry_settings_has_async_callback_invalid_target(self):
ir = InvokeRetry(DummyInvokingService())
kwargs = {
'async_fallback': True,
'callback': rand_string(),
'context': {},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
}
try:
ir._get_retry_settings(rand_string(), **kwargs)
except ValueError, e:
prefix = 'Service:`{}` does not exist, e:`Traceback (most recent call last):'.format(
kwargs['callback'])
self.assertTrue(e.message.startswith(prefix))
self.assertIn("KeyError: u'{}'".format(kwargs['callback']),
e.message)
def test_get_retry_settings_has_async_callback_both_secs_mins(self):
ir = InvokeRetry(None)
target = rand_string()
kwargs = {
'async_fallback': True,
'callback': rand_string(),
'context': {},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': rand_int(),
}
try:
ir._get_retry_settings(target, **kwargs)
except ValueError as e:
self.assertEquals(
e.message,
'Could not invoke `{}`, only one of seconds:`{}` and minutes:`{}` can be given'
.format(target, kwargs['seconds'], kwargs['minutes']))
else:
self.fail('Expected a ValueError')
def test_invoke_async_retry(self):
target = 'target_{}'.format(rand_string())
callback = 'callback_{}'.format(rand_string())
callback_impl_name = 'callback_impl_name_{}'.format(rand_string())
cid = new_cid()
invoking_service = DummyTargetService(callback, callback_impl_name, cid)
ir = InvokeRetry(invoking_service)
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {rand_string():rand_string()},
'repeats': rand_int(),
'seconds': rand_int(),
'minutes': 0,
'cid': cid
}
kwargs_copy = deepcopy(kwargs)
call_cid = ir.invoke_async(target, **kwargs)
self.assertTrue(len(invoking_service.invoke_async_args), 2)
self.assertEquals(invoking_service.invoke_async_kwargs, {'cid': cid})
invoke_retry_service, retry_request = invoking_service.invoke_async_args
self.assertEquals(invoke_retry_service, 'zato.pattern.invoke-retry.invoke-retry')
retry_request = loads(retry_request)
self.assertEquals(retry_request['target'], target)
self.assertEquals(retry_request['callback'], callback)
self.assertEquals(retry_request['args'], [])
self.assertEquals(retry_request['retry_seconds'], kwargs_copy['seconds'])
self.assertEquals(retry_request['kwargs'], {'cid': call_cid})
self.assertEquals(retry_request['orig_cid'], cid)
self.assertEquals(retry_request['retry_repeats'], kwargs_copy['repeats'])
self.assertEquals(retry_request['callback_context'], kwargs_copy['context'])
def test_get_retry_settings_has_async_callback(self):
callback = rand_string()
callback_impl_name = rand_string()
invoking_service = DummyTargetService(callback, callback_impl_name)
for needs_seconds in True, False:
target = rand_string()
kwargs = {
'async_fallback': True,
'callback': callback,
'context': {rand_string():rand_string(), rand_int():rand_int()},
'repeats': rand_int(),
'seconds': rand_int() if needs_seconds else 0,
'minutes': rand_int() if not needs_seconds else 0,
'custom1': 'custom1',
'custom2': 'custom2',
}
kwargs_copy = deepcopy(kwargs)
ir = InvokeRetry(invoking_service)
async_fallback, callback, callback_context, retry_repeats, retry_seconds, kwargs = ir._get_retry_settings(
target, **kwargs)
self.assertTrue(async_fallback)
self.assertNotIn('callback', kwargs)
self.assertEquals(callback, kwargs_copy['callback'])
self.assertNotIn('context', kwargs)
self.assertDictEqual(callback_context, kwargs_copy['context'])
self.assertNotIn('repeats', kwargs)
self.assertEquals(retry_repeats, kwargs_copy['repeats'])
if needs_seconds:
self.assertNotIn('seconds', kwargs)
self.assertEquals(retry_seconds, kwargs_copy['seconds'])
else:
self.assertNotIn('minutes', kwargs)
self.assertEquals(retry_seconds, kwargs_copy['minutes'] * 60)
self.assertDictEqual(kwargs, {'custom1':'custom1', 'custom2':'custom2'})
def __init__(self, invoking_service):
self.invoke_retry = InvokeRetry(invoking_service)
self.fanout = FanOut(invoking_service)
self.parallel = ParallelExec(invoking_service)
def __init__(self, invoking_service):
self._invoke_retry = InvokeRetry(invoking_service)
# Convenience API
self.invoke_retry = self._invoke_retry.invoke_retry
self.invoke_async_retry = self._invoke_retry.invoke_async_retry