def test_ignores_errors(self): """ Errors are not logged and are propogated """ seq = [("internal", lambda i: raise_(ValueError("oops")))] self.assertRaises(ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp) self.assertFalse(self.log.msg.called)
def test_logs_msg(self): """ Logs msg with time and returns result of internal effect """ seq = [("internal", lambda i: self.clock.advance(3) or "result")] self.assertEqual(perform_sequence(seq, msg_with_time("mt", Effect("internal")), self.disp), "result") self.log.msg.assert_called_once_with("mt", seconds_taken=3.0)
def test_logs_msg(self): """ Logs msg with time and returns result of internal effect """ seq = [("internal", lambda i: self.clock.advance(3) or "result")] self.assertEqual( perform_sequence(seq, msg_with_time("mt", Effect("internal")), self.disp), "result") self.log.msg.assert_called_once_with("mt", seconds_taken=3.0)
def execute_convergence(tenant_id, group_id, build_timeout, waiting, limited_retry_iterations, step_limits, get_executor=get_executor): """ Gather data, plan a convergence, save active and pending servers to the group state, and then execute the convergence. :param str tenant_id: the tenant ID for the group to converge :param str group_id: the ID of the group to be converged :param number build_timeout: number of seconds to wait for servers to be in building before it's is timed out and deleted :param Reference waiting: pmap of waiting groups :param int limited_retry_iterations: number of iterations to wait for LIMITED_RETRY steps :param dict step_limits: Mapping of step class to number of executions allowed in a convergence cycle :param callable get_executor: like :func`get_executor`, used for testing. :return: Effect of :obj:`ConvergenceIterationStatus`. :raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist. """ clean_waiting = _clean_waiting(waiting, group_id) # Gather data yield msg("begin-convergence") now_dt = yield Effect(Func(datetime.utcnow)) all_data = yield msg_with_time( "gather-convergence-data", convergence_exec_data(tenant_id, group_id, now_dt, get_executor=get_executor)) (executor, scaling_group, group_state, desired_group_state, resources) = all_data # prepare plan steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt), build_timeout, step_limits, **resources) yield log_steps(steps) # Execute plan yield msg('execute-convergence', steps=steps, now=now_dt, desired=desired_group_state, **resources) worst_status, reasons = yield _execute_steps(steps) if worst_status != StepResult.LIMITED_RETRY: # If we're not waiting any more, there's no point in keeping track of # the group yield clean_waiting # Handle the status from execution if worst_status == StepResult.SUCCESS: result = yield convergence_succeeded(executor, scaling_group, group_state, resources, now_dt) elif worst_status == StepResult.FAILURE: result = yield convergence_failed(scaling_group, reasons) elif worst_status is StepResult.LIMITED_RETRY: # We allow further iterations to proceed as long as we haven't been # waiting for a LIMITED_RETRY for N consecutive iterations. current_iterations = (yield waiting.read()).get(group_id, 0) if current_iterations > limited_retry_iterations: yield msg('converge-limited-retry-too-long') yield clean_waiting # Prefix "Timed out" to all limited retry reasons result = yield convergence_failed(scaling_group, reasons, True) else: yield waiting.modify(lambda group_iterations: group_iterations.set( group_id, current_iterations + 1)) result = ConvergenceIterationStatus.Continue() else: result = ConvergenceIterationStatus.Continue() yield do_return(result)
def execute_convergence(tenant_id, group_id, build_timeout, waiting, limited_retry_iterations, step_limits, get_executor=get_executor): """ Gather data, plan a convergence, save active and pending servers to the group state, and then execute the convergence. :param str tenant_id: the tenant ID for the group to converge :param str group_id: the ID of the group to be converged :param number build_timeout: number of seconds to wait for servers to be in building before it's is timed out and deleted :param Reference waiting: pmap of waiting groups :param int limited_retry_iterations: number of iterations to wait for LIMITED_RETRY steps :param dict step_limits: Mapping of step class to number of executions allowed in a convergence cycle :param callable get_executor: like :func`get_executor`, used for testing. :return: Effect of :obj:`ConvergenceIterationStatus`. :raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist. """ clean_waiting = _clean_waiting(waiting, group_id) # Begin convergence by updating group status to ACTIVE yield msg("begin-convergence") try: yield Effect(LoadAndUpdateGroupStatus(tenant_id, group_id, ScalingGroupStatus.ACTIVE)) except NoSuchScalingGroupError: # Expected for DELETING group. Ignore. pass # Gather data now_dt = yield Effect(Func(datetime.utcnow)) try: all_data = yield msg_with_time( "gather-convergence-data", convergence_exec_data(tenant_id, group_id, now_dt, get_executor=get_executor)) (executor, scaling_group, group_state, desired_group_state, resources) = all_data except FirstError as fe: if fe.exc_info[0] is NoSuchEndpoint: result = yield convergence_failed( tenant_id, group_id, [ErrorReason.Exception(fe.exc_info)]) yield do_return(result) raise fe # prepare plan steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt), build_timeout, step_limits, **resources) yield log_steps(steps) # Execute plan yield msg('execute-convergence', steps=steps, now=now_dt, desired=desired_group_state, **resources) worst_status, reasons = yield _execute_steps(steps) if worst_status != StepResult.LIMITED_RETRY: # If we're not waiting any more, there's no point in keeping track of # the group yield clean_waiting # Handle the status from execution if worst_status == StepResult.SUCCESS: result = yield convergence_succeeded( executor, scaling_group, group_state, resources) elif worst_status == StepResult.FAILURE: result = yield convergence_failed(tenant_id, group_id, reasons) elif worst_status is StepResult.LIMITED_RETRY: # We allow further iterations to proceed as long as we haven't been # waiting for a LIMITED_RETRY for N consecutive iterations. current_iterations = (yield waiting.read()).get(group_id, 0) if current_iterations > limited_retry_iterations: yield msg('converge-limited-retry-too-long') yield clean_waiting # Prefix "Timed out" to all limited retry reasons result = yield convergence_failed(tenant_id, group_id, reasons, True) else: yield waiting.modify( lambda group_iterations: group_iterations.set(group_id, current_iterations + 1)) result = ConvergenceIterationStatus.Continue() else: result = ConvergenceIterationStatus.Continue() yield do_return(result)
def execute_convergence(tenant_id, group_id, build_timeout, waiting, limited_retry_iterations, get_executor=get_executor): """ Gather data, plan a convergence, save active and pending servers to the group state, and then execute the convergence. :param str tenant_id: the tenant ID for the group to converge :param str group_id: the ID of the group to be converged :param number build_timeout: number of seconds to wait for servers to be in building before it's is timed out and deleted :param Reference waiting: pmap of waiting groups :param int limited_retry_iterations: number of iterations to wait for LIMITED_RETRY steps :param callable get_all_convergence_data: like :func`get_all_convergence_data`, used for testing. :param callable plan: like :func:`plan`, to be used for test injection only :return: Effect of :obj:`ConvergenceIterationStatus`. :raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist. """ clean_waiting = _clean_waiting(waiting, group_id) # Gather data yield msg("begin-convergence") now_dt = yield Effect(Func(datetime.utcnow)) all_data = yield msg_with_time( "gather-convergence-data", convergence_exec_data(tenant_id, group_id, now_dt, get_executor=get_executor)) (executor, scaling_group, group_state, desired_group_state, resources) = all_data # prepare plan steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt), build_timeout, **resources) yield log_steps(steps) # Execute plan yield msg('execute-convergence', steps=steps, now=now_dt, desired=desired_group_state, **resources) worst_status, reasons = yield _execute_steps(steps) if worst_status != StepResult.LIMITED_RETRY: # If we're not waiting any more, there's no point in keeping track of # the group yield clean_waiting # Handle the status from execution if worst_status == StepResult.SUCCESS: result = yield convergence_succeeded( executor, scaling_group, group_state, resources, now_dt) elif worst_status == StepResult.FAILURE: result = yield convergence_failed(scaling_group, reasons) elif worst_status is StepResult.LIMITED_RETRY: # We allow further iterations to proceed as long as we haven't been # waiting for a LIMITED_RETRY for N consecutive iterations. current_iterations = (yield waiting.read()).get(group_id, 0) if current_iterations > limited_retry_iterations: yield msg('converge-limited-retry-too-long') yield clean_waiting result = yield convergence_failed(scaling_group, reasons) else: yield waiting.modify( lambda group_iterations: group_iterations.set(group_id, current_iterations + 1)) result = ConvergenceIterationStatus.Continue() else: result = ConvergenceIterationStatus.Continue() yield do_return(result)