Beispiel #1
0
def retry_effect_with_timeout(effect,
                              timeout,
                              retry_wait=timedelta(seconds=1),
                              backoff=True,
                              time=time.time):
    """
    If ``effect`` fails, retry it until ``timeout`` expires.

    To avoid excessive retrying, this function uses the exponential backoff
    algorithm by default, waiting double the time between each retry.

    :param Effect effect: The Effect to retry.
    :param int timeout: Keep retrying until timeout.  This is measured in
        seconds from the time of the first failure of ``effect``.
    :param timedelta retry_wait: The wait time between retries
    :param bool backoff: Whether we should use exponential backoff
    :param callable time: A nullary callable that returns a UNIX timestamp.

    :return: An Effect that does what ``effect`` does, but retrying on
        exception.
    """
    class State(object):
        end_time = None
        wait_time = None

    def should_retry(exc_info):
        # This is the wrong time to compute end_time.  It's a lot simpler to do
        # this than to hook into the effect being wrapped and record the time
        # it starts to run.  Perhaps implementing that would be a nice thing to
        # do later.
        #
        # Anyway, make note of when we want to be finished if we haven't yet
        # done so.
        if State.end_time is None:
            State.end_time = time() + timeout

        if time() >= State.end_time:
            return Effect(Constant(False))
        else:
            retry_delay = State.wait_time.total_seconds()
            effect = Effect(Delay(retry_delay)).on(
                success=lambda x: Effect(Constant(True)))

            if backoff:
                State.wait_time *= 2

            return effect

    State.wait_time = retry_wait

    return retry(effect, should_retry)
Beispiel #2
0
def retry_effect_with_timeout(effect, timeout, retry_wait=timedelta(seconds=1),
                              backoff=True, time=time.time):
    """
    If ``effect`` fails, retry it until ``timeout`` expires.

    To avoid excessive retrying, this function uses the exponential backoff
    algorithm by default, waiting double the time between each retry.

    :param Effect effect: The Effect to retry.
    :param int timeout: Keep retrying until timeout.  This is measured from the
        time of the first failure of ``effect``.
    :param timedelta retry_wait: The wait time between retries
    :param bool backoff: Whether we should use exponential backoff
    :param callable time: A nullary callable that returns a UNIX timestamp.

    :return: An Effect that does what ``effect`` does, but retrying on
        exception.
    """
    class State(object):
        end_time = None
        wait_time = None

    def should_retry(exc_info):
        # This is the wrong time to compute end_time.  It's a lot simpler to do
        # this than to hook into the effect being wrapped and record the time
        # it starts to run.  Perhaps implementing that would be a nice thing to
        # do later.
        #
        # Anyway, make note of when we want to be finished if we haven't yet
        # done so.
        if State.end_time is None:
            State.end_time = time() + timeout

        if time() >= State.end_time:
            return Effect(Constant(False))
        else:
            retry_delay = State.wait_time.total_seconds()
            effect = Effect(Delay(retry_delay)).on(
                success=lambda x: Effect(Constant(True))
            )

            if backoff:
                State.wait_time *= 2

            return effect

    State.wait_time = retry_wait

    return retry(effect, should_retry)
Beispiel #3
0
    def provision(self, package_source, variants=()):
        """
        Provision flocker on this node.

        :param LibcloudNode node: Node to provision.
        :param PackageSource package_source: See func:`task_install_flocker`
        :param set variants: The set of variant configurations to use when
            provisioning
        """
        username = self.get_default_username()

        commands = []

        # cloud-init may not have allowed sudo without tty yet, so try SSH key
        # installation for a few more seconds:
        start = []

        def for_thirty_seconds(*args, **kwargs):
            if not start:
                start.append(time())
            return Effect(Constant((time() - start[0]) < 30))

        commands.append(
            run_remotely(
                username=username,
                address=self.address,
                commands=retry(task_install_ssh_key(), for_thirty_seconds),
            ))

        commands.append(
            run_remotely(
                username='******',
                address=self.address,
                commands=provision(
                    package_source=package_source,
                    distribution=self.distribution,
                    variants=variants,
                ),
            ))

        return sequence(commands)
Beispiel #4
0
def provision_aws(node, package_source, distribution, variants):
    """
    Provision flocker on this node.

    :param LibcloudNode node: Node to provision.
    :param PackageSource package_source: See func:`task_install_flocker`
    :param bytes distribution: See func:`task_install_flocker`
    :param set variants: The set of variant configurations to use when
        provisioning
    """
    username = get_default_username(distribution)

    commands = []

    # cloud-init may not have allowed sudo without tty yet, so try SSH key
    # installation for a few more seconds:
    start = []

    def for_ten_seconds(*args, **kwargs):
        if not start:
            start.append(time())
        return Effect(Constant((time() - start[0]) < 30))

    commands.append(run_remotely(
        username=username,
        address=node.address,
        commands=retry(task_install_ssh_key(), for_ten_seconds),
    ))

    commands.append(run_remotely(
        username='******',
        address=node.address,
        commands=provision(
            package_source=package_source,
            distribution=node.distribution,
            variants=variants,
        ),
    ))

    return sequence(commands)
Beispiel #5
0
def retry_effect_with_timeout(effect,
                              timeout,
                              retry_wait=timedelta(seconds=1),
                              backoff=True,
                              time=time.time):
    """
    If ``effect`` fails, retry it until ``timeout`` expires.

    To avoid excessive retrying, this function uses the exponential backoff
    algorithm by default, waiting double the time between each retry.

    :param Effect effect: The Effect to retry.
    :param int timeout: Keep retrying until timeout.
    :param timedelta retry_wait: The wait time between retries
    :param bool backoff: Whether we should use exponential backoff
    :param callable time: A nullary callable that returns a UNIX timestamp.

    :return: An Effect that does what ``effect`` does, but retrying on
        exception.
    """
    end_time = time() + timeout

    def should_retry(e):
        if time() >= end_time:
            return Effect(Constant(False))
        else:
            retry_delay = should_retry.wait_secs.total_seconds()
            effect = Effect(Delay(retry_delay)).on(
                success=lambda x: Effect(Constant(True)))

            if backoff:
                should_retry.wait_secs *= 2

            return effect

    should_retry.wait_secs = retry_wait

    return retry(effect, should_retry)
Beispiel #6
0
def retry_effect_with_timeout(effect, timeout, retry_wait=timedelta(seconds=1),
                              backoff=True, time=time.time):
    """
    If ``effect`` fails, retry it until ``timeout`` expires.

    To avoid excessive retrying, this function uses the exponential backoff
    algorithm by default, waiting double the time between each retry.

    :param Effect effect: The Effect to retry.
    :param int timeout: Keep retrying until timeout.
    :param timedelta retry_wait: The wait time between retries
    :param bool backoff: Whether we should use exponential backoff
    :param callable time: A nullary callable that returns a UNIX timestamp.

    :return: An Effect that does what ``effect`` does, but retrying on
        exception.
    """
    end_time = time() + timeout

    def should_retry(e):
        if time() >= end_time:
            return Effect(Constant(False))
        else:
            retry_delay = should_retry.wait_secs.total_seconds()
            effect = Effect(Delay(retry_delay)).on(
                success=lambda x: Effect(Constant(True))
            )

            if backoff:
                should_retry.wait_secs *= 2

            return effect

    should_retry.wait_secs = retry_wait

    return retry(effect, should_retry)