Exemplo n.º 1
0
 def test_ignores_errors(self):
     """
     Errors are not logged and are propogated
     """
     seq = [("internal", lambda i: raise_(ValueError("oops")))]
     self.assertRaises(ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp)
     self.assertFalse(self.log.msg.called)
Exemplo n.º 2
0
 def test_logs_msg(self):
     """
     Logs msg with time and returns result of internal effect
     """
     seq = [("internal", lambda i: self.clock.advance(3) or "result")]
     self.assertEqual(perform_sequence(seq, msg_with_time("mt", Effect("internal")), self.disp), "result")
     self.log.msg.assert_called_once_with("mt", seconds_taken=3.0)
Exemplo n.º 3
0
 def test_ignores_errors(self):
     """
     Errors are not logged and are propogated
     """
     seq = [("internal", lambda i: raise_(ValueError("oops")))]
     self.assertRaises(ValueError, perform_sequence, seq,
                       msg_with_time("mt", Effect("internal")), self.disp)
     self.assertFalse(self.log.msg.called)
Exemplo n.º 4
0
 def test_logs_msg(self):
     """
     Logs msg with time and returns result of internal effect
     """
     seq = [("internal", lambda i: self.clock.advance(3) or "result")]
     self.assertEqual(
         perform_sequence(seq, msg_with_time("mt", Effect("internal")),
                          self.disp), "result")
     self.log.msg.assert_called_once_with("mt", seconds_taken=3.0)
Exemplo n.º 5
0
def execute_convergence(tenant_id,
                        group_id,
                        build_timeout,
                        waiting,
                        limited_retry_iterations,
                        step_limits,
                        get_executor=get_executor):
    """
    Gather data, plan a convergence, save active and pending servers to the
    group state, and then execute the convergence.

    :param str tenant_id: the tenant ID for the group to converge
    :param str group_id: the ID of the group to be converged
    :param number build_timeout: number of seconds to wait for servers to be in
        building before it's is timed out and deleted
    :param Reference waiting: pmap of waiting groups
    :param int limited_retry_iterations: number of iterations to wait for
        LIMITED_RETRY steps
    :param dict step_limits: Mapping of step class to number of executions
        allowed in a convergence cycle
    :param callable get_executor: like :func`get_executor`, used for testing.

    :return: Effect of :obj:`ConvergenceIterationStatus`.
    :raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist.
    """
    clean_waiting = _clean_waiting(waiting, group_id)
    # Gather data
    yield msg("begin-convergence")
    now_dt = yield Effect(Func(datetime.utcnow))
    all_data = yield msg_with_time(
        "gather-convergence-data",
        convergence_exec_data(tenant_id,
                              group_id,
                              now_dt,
                              get_executor=get_executor))
    (executor, scaling_group, group_state, desired_group_state,
     resources) = all_data

    # prepare plan
    steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt),
                          build_timeout, step_limits, **resources)
    yield log_steps(steps)

    # Execute plan
    yield msg('execute-convergence',
              steps=steps,
              now=now_dt,
              desired=desired_group_state,
              **resources)
    worst_status, reasons = yield _execute_steps(steps)

    if worst_status != StepResult.LIMITED_RETRY:
        # If we're not waiting any more, there's no point in keeping track of
        # the group
        yield clean_waiting

    # Handle the status from execution
    if worst_status == StepResult.SUCCESS:
        result = yield convergence_succeeded(executor, scaling_group,
                                             group_state, resources, now_dt)
    elif worst_status == StepResult.FAILURE:
        result = yield convergence_failed(scaling_group, reasons)
    elif worst_status is StepResult.LIMITED_RETRY:
        # We allow further iterations to proceed as long as we haven't been
        # waiting for a LIMITED_RETRY for N consecutive iterations.
        current_iterations = (yield waiting.read()).get(group_id, 0)
        if current_iterations > limited_retry_iterations:
            yield msg('converge-limited-retry-too-long')
            yield clean_waiting
            # Prefix "Timed out" to all limited retry reasons
            result = yield convergence_failed(scaling_group, reasons, True)
        else:
            yield waiting.modify(lambda group_iterations: group_iterations.set(
                group_id, current_iterations + 1))
            result = ConvergenceIterationStatus.Continue()
    else:
        result = ConvergenceIterationStatus.Continue()
    yield do_return(result)
Exemplo n.º 6
0
def execute_convergence(tenant_id, group_id, build_timeout, waiting,
                        limited_retry_iterations, step_limits,
                        get_executor=get_executor):
    """
    Gather data, plan a convergence, save active and pending servers to the
    group state, and then execute the convergence.

    :param str tenant_id: the tenant ID for the group to converge
    :param str group_id: the ID of the group to be converged
    :param number build_timeout: number of seconds to wait for servers to be in
        building before it's is timed out and deleted
    :param Reference waiting: pmap of waiting groups
    :param int limited_retry_iterations: number of iterations to wait for
        LIMITED_RETRY steps
    :param dict step_limits: Mapping of step class to number of executions
        allowed in a convergence cycle
    :param callable get_executor: like :func`get_executor`, used for testing.

    :return: Effect of :obj:`ConvergenceIterationStatus`.
    :raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist.
    """
    clean_waiting = _clean_waiting(waiting, group_id)

    # Begin convergence by updating group status to ACTIVE
    yield msg("begin-convergence")
    try:
        yield Effect(LoadAndUpdateGroupStatus(tenant_id, group_id,
                                              ScalingGroupStatus.ACTIVE))
    except NoSuchScalingGroupError:
        # Expected for DELETING group. Ignore.
        pass

    # Gather data
    now_dt = yield Effect(Func(datetime.utcnow))
    try:
        all_data = yield msg_with_time(
            "gather-convergence-data",
            convergence_exec_data(tenant_id, group_id, now_dt,
                                  get_executor=get_executor))
        (executor, scaling_group, group_state, desired_group_state,
         resources) = all_data
    except FirstError as fe:
        if fe.exc_info[0] is NoSuchEndpoint:
            result = yield convergence_failed(
                tenant_id, group_id, [ErrorReason.Exception(fe.exc_info)])
            yield do_return(result)
        raise fe

    # prepare plan
    steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt),
                          build_timeout, step_limits, **resources)
    yield log_steps(steps)

    # Execute plan
    yield msg('execute-convergence',
              steps=steps, now=now_dt, desired=desired_group_state,
              **resources)
    worst_status, reasons = yield _execute_steps(steps)

    if worst_status != StepResult.LIMITED_RETRY:
        # If we're not waiting any more, there's no point in keeping track of
        # the group
        yield clean_waiting

    # Handle the status from execution
    if worst_status == StepResult.SUCCESS:
        result = yield convergence_succeeded(
            executor, scaling_group, group_state, resources)
    elif worst_status == StepResult.FAILURE:
        result = yield convergence_failed(tenant_id, group_id, reasons)
    elif worst_status is StepResult.LIMITED_RETRY:
        # We allow further iterations to proceed as long as we haven't been
        # waiting for a LIMITED_RETRY for N consecutive iterations.
        current_iterations = (yield waiting.read()).get(group_id, 0)
        if current_iterations > limited_retry_iterations:
            yield msg('converge-limited-retry-too-long')
            yield clean_waiting
            # Prefix "Timed out" to all limited retry reasons
            result = yield convergence_failed(tenant_id, group_id, reasons,
                                              True)
        else:
            yield waiting.modify(
                lambda group_iterations:
                    group_iterations.set(group_id, current_iterations + 1))
            result = ConvergenceIterationStatus.Continue()
    else:
        result = ConvergenceIterationStatus.Continue()
    yield do_return(result)
Exemplo n.º 7
0
def execute_convergence(tenant_id, group_id, build_timeout, waiting,
                        limited_retry_iterations, get_executor=get_executor):
    """
    Gather data, plan a convergence, save active and pending servers to the
    group state, and then execute the convergence.

    :param str tenant_id: the tenant ID for the group to converge
    :param str group_id: the ID of the group to be converged
    :param number build_timeout: number of seconds to wait for servers to be in
        building before it's is timed out and deleted
    :param Reference waiting: pmap of waiting groups
    :param int limited_retry_iterations: number of iterations to wait for
        LIMITED_RETRY steps
    :param callable get_all_convergence_data: like
        :func`get_all_convergence_data`, used for testing.
    :param callable plan: like :func:`plan`, to be used for test injection only

    :return: Effect of :obj:`ConvergenceIterationStatus`.
    :raise: :obj:`NoSuchScalingGroupError` if the group doesn't exist.
    """
    clean_waiting = _clean_waiting(waiting, group_id)
    # Gather data
    yield msg("begin-convergence")
    now_dt = yield Effect(Func(datetime.utcnow))
    all_data = yield msg_with_time(
        "gather-convergence-data",
        convergence_exec_data(tenant_id, group_id, now_dt,
                              get_executor=get_executor))
    (executor, scaling_group, group_state, desired_group_state,
     resources) = all_data

    # prepare plan
    steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt),
                          build_timeout, **resources)
    yield log_steps(steps)

    # Execute plan
    yield msg('execute-convergence',
              steps=steps, now=now_dt, desired=desired_group_state,
              **resources)
    worst_status, reasons = yield _execute_steps(steps)

    if worst_status != StepResult.LIMITED_RETRY:
        # If we're not waiting any more, there's no point in keeping track of
        # the group
        yield clean_waiting

    # Handle the status from execution
    if worst_status == StepResult.SUCCESS:
        result = yield convergence_succeeded(
            executor, scaling_group, group_state, resources, now_dt)
    elif worst_status == StepResult.FAILURE:
        result = yield convergence_failed(scaling_group, reasons)
    elif worst_status is StepResult.LIMITED_RETRY:
        # We allow further iterations to proceed as long as we haven't been
        # waiting for a LIMITED_RETRY for N consecutive iterations.
        current_iterations = (yield waiting.read()).get(group_id, 0)
        if current_iterations > limited_retry_iterations:
            yield msg('converge-limited-retry-too-long')
            yield clean_waiting
            result = yield convergence_failed(scaling_group, reasons)
        else:
            yield waiting.modify(
                lambda group_iterations:
                    group_iterations.set(group_id, current_iterations + 1))
            result = ConvergenceIterationStatus.Continue()
    else:
        result = ConvergenceIterationStatus.Continue()
    yield do_return(result)