def test_ignores_errors(self):
"""
Errors are not logged and are propogated
"""
seq = [("internal", lambda i: raise_(ValueError("oops")))]
self.assertRaises(ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp)
self.assertFalse(self.log.msg.called)
def test_logs_msg(self):
"""
Logs msg with time and returns result of internal effect
"""
seq = [("internal", lambda i: self.clock.advance(3) or "result")]
self.assertEqual(perform_sequence(seq, msg_with_time("mt", Effect("internal")), self.disp), "result")
self.log.msg.assert_called_once_with("mt", seconds_taken=3.0)
def test_ignores_errors(self):
"""
Errors are not logged and are propogated
"""
seq = [("internal", lambda i: raise_(ValueError("oops")))]
self.assertRaises(ValueError, perform_sequence, seq,
msg_with_time("mt", Effect("internal")), self.disp)
self.assertFalse(self.log.msg.called)
def test_logs_msg(self):
"""
Logs msg with time and returns result of internal effect
"""
seq = [("internal", lambda i: self.clock.advance(3) or "result")]
self.assertEqual(
perform_sequence(seq, msg_with_time("mt", Effect("internal")),
self.disp), "result")
self.log.msg.assert_called_once_with("mt", seconds_taken=3.0)
def execute_convergence(tenant_id,
group_id,
build_timeout,
waiting,
limited_retry_iterations,
step_limits,
get_executor=get_executor):
"""
Gather data, plan a convergence, save active and pending servers to the
group state, and then execute the convergence.
:param str tenant_id: the tenant ID for the group to converge
:param str group_id: the ID of the group to be converged
:param number build_timeout: number of seconds to wait for servers to be in
building before it's is timed out and deleted
:param Reference waiting: pmap of waiting groups
:param int limited_retry_iterations: number of iterations to wait for
LIMITED_RETRY steps
:param dict step_limits: Mapping of step class to number of executions
allowed in a convergence cycle
:param callable get_executor: like :func`get_executor`, used for testing.
:return: Effect of :obj:`ConvergenceIterationStatus`.
:raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist.
"""
clean_waiting = _clean_waiting(waiting, group_id)
# Gather data
yield msg("begin-convergence")
now_dt = yield Effect(Func(datetime.utcnow))
all_data = yield msg_with_time(
"gather-convergence-data",
convergence_exec_data(tenant_id,
group_id,
now_dt,
get_executor=get_executor))
(executor, scaling_group, group_state, desired_group_state,
resources) = all_data
# prepare plan
steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt),
build_timeout, step_limits, **resources)
yield log_steps(steps)
# Execute plan
yield msg('execute-convergence',
steps=steps,
now=now_dt,
desired=desired_group_state,
**resources)
worst_status, reasons = yield _execute_steps(steps)
if worst_status != StepResult.LIMITED_RETRY:
# If we're not waiting any more, there's no point in keeping track of
# the group
yield clean_waiting
# Handle the status from execution
if worst_status == StepResult.SUCCESS:
result = yield convergence_succeeded(executor, scaling_group,
group_state, resources, now_dt)
elif worst_status == StepResult.FAILURE:
result = yield convergence_failed(scaling_group, reasons)
elif worst_status is StepResult.LIMITED_RETRY:
# We allow further iterations to proceed as long as we haven't been
# waiting for a LIMITED_RETRY for N consecutive iterations.
current_iterations = (yield waiting.read()).get(group_id, 0)
if current_iterations > limited_retry_iterations:
yield msg('converge-limited-retry-too-long')
yield clean_waiting
# Prefix "Timed out" to all limited retry reasons
result = yield convergence_failed(scaling_group, reasons, True)
else:
yield waiting.modify(lambda group_iterations: group_iterations.set(
group_id, current_iterations + 1))
result = ConvergenceIterationStatus.Continue()
else:
result = ConvergenceIterationStatus.Continue()
yield do_return(result)
def execute_convergence(tenant_id, group_id, build_timeout, waiting,
limited_retry_iterations, step_limits,
get_executor=get_executor):
"""
Gather data, plan a convergence, save active and pending servers to the
group state, and then execute the convergence.
:param str tenant_id: the tenant ID for the group to converge
:param str group_id: the ID of the group to be converged
:param number build_timeout: number of seconds to wait for servers to be in
building before it's is timed out and deleted
:param Reference waiting: pmap of waiting groups
:param int limited_retry_iterations: number of iterations to wait for
LIMITED_RETRY steps
:param dict step_limits: Mapping of step class to number of executions
allowed in a convergence cycle
:param callable get_executor: like :func`get_executor`, used for testing.
:return: Effect of :obj:`ConvergenceIterationStatus`.
:raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist.
"""
clean_waiting = _clean_waiting(waiting, group_id)
# Begin convergence by updating group status to ACTIVE
yield msg("begin-convergence")
try:
yield Effect(LoadAndUpdateGroupStatus(tenant_id, group_id,
ScalingGroupStatus.ACTIVE))
except NoSuchScalingGroupError:
# Expected for DELETING group. Ignore.
pass
# Gather data
now_dt = yield Effect(Func(datetime.utcnow))
try:
all_data = yield msg_with_time(
"gather-convergence-data",
convergence_exec_data(tenant_id, group_id, now_dt,
get_executor=get_executor))
(executor, scaling_group, group_state, desired_group_state,
resources) = all_data
except FirstError as fe:
if fe.exc_info[0] is NoSuchEndpoint:
result = yield convergence_failed(
tenant_id, group_id, [ErrorReason.Exception(fe.exc_info)])
yield do_return(result)
raise fe
# prepare plan
steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt),
build_timeout, step_limits, **resources)
yield log_steps(steps)
# Execute plan
yield msg('execute-convergence',
steps=steps, now=now_dt, desired=desired_group_state,
**resources)
worst_status, reasons = yield _execute_steps(steps)
if worst_status != StepResult.LIMITED_RETRY:
# If we're not waiting any more, there's no point in keeping track of
# the group
yield clean_waiting
# Handle the status from execution
if worst_status == StepResult.SUCCESS:
result = yield convergence_succeeded(
executor, scaling_group, group_state, resources)
elif worst_status == StepResult.FAILURE:
result = yield convergence_failed(tenant_id, group_id, reasons)
elif worst_status is StepResult.LIMITED_RETRY:
# We allow further iterations to proceed as long as we haven't been
# waiting for a LIMITED_RETRY for N consecutive iterations.
current_iterations = (yield waiting.read()).get(group_id, 0)
if current_iterations > limited_retry_iterations:
yield msg('converge-limited-retry-too-long')
yield clean_waiting
# Prefix "Timed out" to all limited retry reasons
result = yield convergence_failed(tenant_id, group_id, reasons,
True)
else:
yield waiting.modify(
lambda group_iterations:
group_iterations.set(group_id, current_iterations + 1))
result = ConvergenceIterationStatus.Continue()
else:
result = ConvergenceIterationStatus.Continue()
yield do_return(result)
def execute_convergence(tenant_id, group_id, build_timeout, waiting,
limited_retry_iterations, get_executor=get_executor):
"""
Gather data, plan a convergence, save active and pending servers to the
group state, and then execute the convergence.
:param str tenant_id: the tenant ID for the group to converge
:param str group_id: the ID of the group to be converged
:param number build_timeout: number of seconds to wait for servers to be in
building before it's is timed out and deleted
:param Reference waiting: pmap of waiting groups
:param int limited_retry_iterations: number of iterations to wait for
LIMITED_RETRY steps
:param callable get_all_convergence_data: like
:func`get_all_convergence_data`, used for testing.
:param callable plan: like :func:`plan`, to be used for test injection only
:return: Effect of :obj:`ConvergenceIterationStatus`.
:raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist.
"""
clean_waiting = _clean_waiting(waiting, group_id)
# Gather data
yield msg("begin-convergence")
now_dt = yield Effect(Func(datetime.utcnow))
all_data = yield msg_with_time(
"gather-convergence-data",
convergence_exec_data(tenant_id, group_id, now_dt,
get_executor=get_executor))
(executor, scaling_group, group_state, desired_group_state,
resources) = all_data
# prepare plan
steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt),
build_timeout, **resources)
yield log_steps(steps)
# Execute plan
yield msg('execute-convergence',
steps=steps, now=now_dt, desired=desired_group_state,
**resources)
worst_status, reasons = yield _execute_steps(steps)
if worst_status != StepResult.LIMITED_RETRY:
# If we're not waiting any more, there's no point in keeping track of
# the group
yield clean_waiting
# Handle the status from execution
if worst_status == StepResult.SUCCESS:
result = yield convergence_succeeded(
executor, scaling_group, group_state, resources, now_dt)
elif worst_status == StepResult.FAILURE:
result = yield convergence_failed(scaling_group, reasons)
elif worst_status is StepResult.LIMITED_RETRY:
# We allow further iterations to proceed as long as we haven't been
# waiting for a LIMITED_RETRY for N consecutive iterations.
current_iterations = (yield waiting.read()).get(group_id, 0)
if current_iterations > limited_retry_iterations:
yield msg('converge-limited-retry-too-long')
yield clean_waiting
result = yield convergence_failed(scaling_group, reasons)
else:
yield waiting.modify(
lambda group_iterations:
group_iterations.set(group_id, current_iterations + 1))
result = ConvergenceIterationStatus.Continue()
else:
result = ConvergenceIterationStatus.Continue()
yield do_return(result)