def converge_one_group(currently_converging, recently_converged, waiting,
tenant_id, group_id, version,
build_timeout, limited_retry_iterations, step_limits,
execute_convergence=execute_convergence):
"""
Converge one group, non-concurrently, and clean up the dirty flag when
done.
:param Reference currently_converging: pset of currently converging groups
:param Reference recently_converged: pmap of recently converged groups
:param Reference waiting: pmap of waiting groups
:param str tenant_id: the tenant ID of the group that is converging
:param str group_id: the ID of the group that is converging
:param version: version number of ZNode of the group's dirty flag
:param number build_timeout: number of seconds to wait for servers to be in
building before it's is timed out and deleted
:param int limited_retry_iterations: number of iterations to wait for
LIMITED_RETRY steps
:param dict step_limits: Mapping of step class to number of executions
allowed in a convergence cycle
:param callable execute_convergence: like :func`execute_convergence`, to
be used for test injection only
"""
mark_recently_converged = Effect(Func(time.time)).on(
lambda time_done: recently_converged.modify(
lambda rcg: rcg.set(group_id, time_done)))
cvg = eff_finally(
execute_convergence(tenant_id, group_id, build_timeout, waiting,
limited_retry_iterations, step_limits),
mark_recently_converged)
try:
result = yield non_concurrently(currently_converging, group_id, cvg)
except ConcurrentError:
# We don't need to spam the logs about this, it's to be expected
return
except NoSuchScalingGroupError:
# NoSuchEndpoint occurs on a suspended or closed account
yield err(None, 'converge-fatal-error')
yield _clean_waiting(waiting, group_id)
yield delete_divergent_flag(tenant_id, group_id, version)
return
except Exception:
# We specifically don't clean up the dirty flag in the case of
# unexpected errors, so convergence will be retried.
yield err(None, 'converge-non-fatal-error')
else:
@match(ConvergenceIterationStatus)
class clean_up(object):
def Continue():
return Effect(Constant(None))
def Stop():
return delete_divergent_flag(tenant_id, group_id, version)
def GroupDeleted():
# Delete the divergent flag to avoid any queued-up convergences
# that will imminently fail.
return delete_divergent_flag(tenant_id, group_id, -1)
yield clean_up(result)
def test_multiple_err(self):
"""
Multiple errors are logged when there are multiple LogErr effects
"""
f1, f2 = object(), object()
eff = err(f1, "yo", a="b").on(lambda _: err(f2, "goo", d="c"))
self.assertIsNone(sync_perform(self.disp, eff))
self.log.err.assert_has_calls([mock.call(f1, "yo", f1="v", a="b"), mock.call(f2, "goo", f1="v", d="c")])
def test_multiple_err(self):
"""
Multiple errors are logged when there are multiple LogErr effects
"""
f1, f2 = object(), object()
eff = err(f1, "yo", a='b').on(lambda _: err(f2, "goo", d='c'))
self.assertIsNone(sync_perform(self.disp, eff))
self.log.err.assert_has_calls([
mock.call(f1, "yo", f1='v', a='b'),
mock.call(f2, "goo", f1='v', d='c')
])
def test_err_from_tuple(self):
"""
exc_info tuple can be passed as failure when constructing LogErr
in which case failure will be constructed from the tuple
"""
eff = err((ValueError, ValueError("a"), None), "why")
sync_perform(self.disp, eff)
self.log.err.assert_called_once_with(CheckFailureValue(ValueError("a")), "why", f1="v")
def test_err(self):
"""
error is logged with original field
"""
f = object()
r = sync_perform(self.disp, err(f, "yo!"))
self.assertIsNone(r)
self.log.err.assert_called_once_with(f, "yo!", f1='v')
def _converge_all(self, my_buckets, divergent_flags):
"""Run :func:`converge_all_groups` and log errors."""
eff = self._converge_all_groups(
self.currently_converging, self.recently_converged, self.waiting,
my_buckets, self._buckets, divergent_flags, self.build_timeout,
self.interval, self.limited_retry_iterations, self.step_limits)
return eff.on(error=lambda e: err(exc_info_to_failure(e),
'converge-all-groups-error'))
def test_err_with_params(self):
"""
error is logged with its fields combined
"""
f = object()
r = sync_perform(self.disp, err(f, "yo!", a='b'))
self.assertIsNone(r)
self.log.err.assert_called_once_with(f, "yo!", f1='v', a='b')
def test_err(self):
"""
error is logged with original field
"""
f = object()
r = sync_perform(self.disp, err(f, "yo!"))
self.assertIsNone(r)
self.log.err.assert_called_once_with(f, "yo!", f1='v')
def test_err_with_params(self):
"""
error is logged with its fields combined
"""
f = object()
r = sync_perform(self.disp, err(f, "yo!", a='b'))
self.assertIsNone(r)
self.log.err.assert_called_once_with(f, "yo!", f1='v', a='b')
def test_nested_err(self):
"""
error is logged when nested inside other effects
"""
f = object()
eff = Effect(Constant("foo")).on(lambda _: err(f, "yo", a="b")).on(lambda _: Effect(Constant("goo")))
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.err.assert_called_once_with(f, "yo", f1="v", a="b")
def test_nested_err(self):
"""
error is logged when nested inside other effects
"""
f = object()
eff = Effect(Constant("foo")).on(lambda _: err(f, "yo", a='b')).on(
lambda _: Effect(Constant("goo")))
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.err.assert_called_once_with(f, "yo", f1='v', a='b')
def _converge_all(self, my_buckets, divergent_flags):
"""Run :func:`converge_all_groups` and log errors."""
eff = self._converge_all_groups(
self.currently_converging, self.recently_converged,
self.waiting,
my_buckets, self._buckets, divergent_flags, self.build_timeout,
self.interval, self.limited_retry_iterations, self.step_limits)
return eff.on(
error=lambda e: err(
exc_info_to_failure(e), 'converge-all-groups-error'))
def test_err_from_tuple(self):
"""
exc_info tuple can be passed as failure when constructing LogErr
in which case failure will be constructed from the tuple
"""
eff = err((ValueError, ValueError("a"), None), "why")
sync_perform(self.disp, eff)
self.log.err.assert_called_once_with(CheckFailureValue(
ValueError('a')),
'why',
f1='v')
def test_boundfields(self):
"""
When an effect is wrapped `BoundFields` then any logging effect
inside is performed with fields setup in `BoundFields`
"""
f = object()
eff = Effect(Constant("foo")).on(lambda _: err(f, "yo", a='b')).on(
lambda _: msg("foo", m='d')).on(lambda _: Effect(Constant("goo")))
eff = with_log(eff, bf='new')
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.msg.assert_called_once_with("foo", f1='v', bf='new', m='d')
self.log.err.assert_called_once_with(f, "yo", f1='v', bf='new', a='b')
def test_boundfields(self):
"""
When an effect is wrapped `BoundFields` then any logging effect
inside is performed with fields setup in `BoundFields`
"""
f = object()
eff = Effect(Constant("foo")).on(
lambda _: err(f, "yo", a='b')).on(
lambda _: msg("foo", m='d')).on(
lambda _: Effect(Constant("goo")))
eff = with_log(eff, bf='new')
self.assertEqual(sync_perform(self.disp, eff), "goo")
self.log.msg.assert_called_once_with("foo", f1='v', bf='new', m='d')
self.log.err.assert_called_once_with(f, "yo", f1='v', bf='new', a='b')
def test_err_from_context(self):
"""
When None is passed as the failure, the exception comes from the
context at the time of creating the intent, not the time at which the
intent is performed.
"""
try:
raise RuntimeError("original")
except RuntimeError:
eff = err(None, "why")
try:
raise RuntimeError("performing")
except RuntimeError:
sync_perform(self.disp, eff)
self.log.err.assert_called_once_with(CheckFailureValue(RuntimeError("original")), "why", f1="v")
def test_err_from_context(self):
"""
When None is passed as the failure, the exception comes from the
context at the time of creating the intent, not the time at which the
intent is performed.
"""
try:
raise RuntimeError('original')
except RuntimeError:
eff = err(None, "why")
try:
raise RuntimeError('performing')
except RuntimeError:
sync_perform(self.disp, eff)
self.log.err.assert_called_once_with(CheckFailureValue(
RuntimeError('original')),
'why',
f1='v')
def delete_divergent_flag(tenant_id, group_id, version):
"""
Delete the dirty flag, if its version hasn't changed. See note [Divergent
flags] for more info.
:return: Effect of None.
"""
flag = format_dirty_flag(tenant_id, group_id)
path = CONVERGENCE_DIRTY_DIR + '/' + flag
fields = dict(path=path, dirty_version=version)
try:
yield Effect(DeleteNode(path=path, version=version))
except BadVersionError:
# BadVersionError shouldn't be logged as an error because it's an
# expected occurrence any time convergence is requested multiple times
# rapidly.
yield msg('mark-clean-skipped', **fields)
except NoNodeError:
yield msg('mark-clean-not-found', **fields)
except Exception:
yield err(None, 'mark-clean-failure', **fields)
else:
yield msg('mark-clean-success')
def delete_divergent_flag(tenant_id, group_id, version):
"""
Delete the dirty flag, if its version hasn't changed. See note [Divergent
flags] for more info.
:return: Effect of None.
"""
flag = format_dirty_flag(tenant_id, group_id)
path = CONVERGENCE_DIRTY_DIR + '/' + flag
fields = dict(path=path, dirty_version=version)
try:
yield Effect(DeleteNode(path=path, version=version))
except BadVersionError:
# BadVersionError shouldn't be logged as an error because it's an
# expected occurrence any time convergence is requested multiple times
# rapidly.
yield msg('mark-clean-skipped', **fields)
except NoNodeError:
yield msg('mark-clean-not-found', **fields)
except Exception:
yield err(None, 'mark-clean-failure', **fields)
else:
yield msg('mark-clean-success')
def log_and_raise(msg, exc_info):
"""
Log error and raise it
"""
eff = err(exc_info_to_failure(exc_info), msg)
return eff.on(lambda _: six.reraise(*exc_info))
def log_and_raise(msg, exc_info):
"""
Log error and raise it
"""
eff = err(exc_info_to_failure(exc_info), msg)
return eff.on(lambda _: six.reraise(*exc_info))
def update_last_info(fname, tenants_len, time):
eff = Effect(
WriteFileLines(
fname, [tenants_len, datetime_to_epoch(time)]))
return eff.on(error=lambda e: err(e, "error updating number of tenants"))
def log_and_return(e):
_eff = err(e, "error reading previous number of tenants")
return _eff.on(lambda _: (None, None))