def retry_effect_with_timeout(effect,
timeout,
retry_wait=timedelta(seconds=1),
backoff=True,
time=time.time):
"""
If ``effect`` fails, retry it until ``timeout`` expires.
To avoid excessive retrying, this function uses the exponential backoff
algorithm by default, waiting double the time between each retry.
:param Effect effect: The Effect to retry.
:param int timeout: Keep retrying until timeout. This is measured in
seconds from the time of the first failure of ``effect``.
:param timedelta retry_wait: The wait time between retries
:param bool backoff: Whether we should use exponential backoff
:param callable time: A nullary callable that returns a UNIX timestamp.
:return: An Effect that does what ``effect`` does, but retrying on
exception.
"""
class State(object):
end_time = None
wait_time = None
def should_retry(exc_info):
# This is the wrong time to compute end_time. It's a lot simpler to do
# this than to hook into the effect being wrapped and record the time
# it starts to run. Perhaps implementing that would be a nice thing to
# do later.
#
# Anyway, make note of when we want to be finished if we haven't yet
# done so.
if State.end_time is None:
State.end_time = time() + timeout
if time() >= State.end_time:
return Effect(Constant(False))
else:
retry_delay = State.wait_time.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
State.wait_time *= 2
return effect
State.wait_time = retry_wait
return retry(effect, should_retry)
def retry_effect_with_timeout(effect, timeout, retry_wait=timedelta(seconds=1),
backoff=True, time=time.time):
"""
If ``effect`` fails, retry it until ``timeout`` expires.
To avoid excessive retrying, this function uses the exponential backoff
algorithm by default, waiting double the time between each retry.
:param Effect effect: The Effect to retry.
:param int timeout: Keep retrying until timeout. This is measured from the
time of the first failure of ``effect``.
:param timedelta retry_wait: The wait time between retries
:param bool backoff: Whether we should use exponential backoff
:param callable time: A nullary callable that returns a UNIX timestamp.
:return: An Effect that does what ``effect`` does, but retrying on
exception.
"""
class State(object):
end_time = None
wait_time = None
def should_retry(exc_info):
# This is the wrong time to compute end_time. It's a lot simpler to do
# this than to hook into the effect being wrapped and record the time
# it starts to run. Perhaps implementing that would be a nice thing to
# do later.
#
# Anyway, make note of when we want to be finished if we haven't yet
# done so.
if State.end_time is None:
State.end_time = time() + timeout
if time() >= State.end_time:
return Effect(Constant(False))
else:
retry_delay = State.wait_time.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True))
)
if backoff:
State.wait_time *= 2
return effect
State.wait_time = retry_wait
return retry(effect, should_retry)
def provision(self, package_source, variants=()):
"""
Provision flocker on this node.
:param LibcloudNode node: Node to provision.
:param PackageSource package_source: See func:`task_install_flocker`
:param set variants: The set of variant configurations to use when
provisioning
"""
username = self.get_default_username()
commands = []
# cloud-init may not have allowed sudo without tty yet, so try SSH key
# installation for a few more seconds:
start = []
def for_thirty_seconds(*args, **kwargs):
if not start:
start.append(time())
return Effect(Constant((time() - start[0]) < 30))
commands.append(
run_remotely(
username=username,
address=self.address,
commands=retry(task_install_ssh_key(), for_thirty_seconds),
))
commands.append(
run_remotely(
username='******',
address=self.address,
commands=provision(
package_source=package_source,
distribution=self.distribution,
variants=variants,
),
))
return sequence(commands)
def provision_aws(node, package_source, distribution, variants):
"""
Provision flocker on this node.
:param LibcloudNode node: Node to provision.
:param PackageSource package_source: See func:`task_install_flocker`
:param bytes distribution: See func:`task_install_flocker`
:param set variants: The set of variant configurations to use when
provisioning
"""
username = get_default_username(distribution)
commands = []
# cloud-init may not have allowed sudo without tty yet, so try SSH key
# installation for a few more seconds:
start = []
def for_ten_seconds(*args, **kwargs):
if not start:
start.append(time())
return Effect(Constant((time() - start[0]) < 30))
commands.append(run_remotely(
username=username,
address=node.address,
commands=retry(task_install_ssh_key(), for_ten_seconds),
))
commands.append(run_remotely(
username='******',
address=node.address,
commands=provision(
package_source=package_source,
distribution=node.distribution,
variants=variants,
),
))
return sequence(commands)
def retry_effect_with_timeout(effect,
timeout,
retry_wait=timedelta(seconds=1),
backoff=True,
time=time.time):
"""
If ``effect`` fails, retry it until ``timeout`` expires.
To avoid excessive retrying, this function uses the exponential backoff
algorithm by default, waiting double the time between each retry.
:param Effect effect: The Effect to retry.
:param int timeout: Keep retrying until timeout.
:param timedelta retry_wait: The wait time between retries
:param bool backoff: Whether we should use exponential backoff
:param callable time: A nullary callable that returns a UNIX timestamp.
:return: An Effect that does what ``effect`` does, but retrying on
exception.
"""
end_time = time() + timeout
def should_retry(e):
if time() >= end_time:
return Effect(Constant(False))
else:
retry_delay = should_retry.wait_secs.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True)))
if backoff:
should_retry.wait_secs *= 2
return effect
should_retry.wait_secs = retry_wait
return retry(effect, should_retry)
def retry_effect_with_timeout(effect, timeout, retry_wait=timedelta(seconds=1),
backoff=True, time=time.time):
"""
If ``effect`` fails, retry it until ``timeout`` expires.
To avoid excessive retrying, this function uses the exponential backoff
algorithm by default, waiting double the time between each retry.
:param Effect effect: The Effect to retry.
:param int timeout: Keep retrying until timeout.
:param timedelta retry_wait: The wait time between retries
:param bool backoff: Whether we should use exponential backoff
:param callable time: A nullary callable that returns a UNIX timestamp.
:return: An Effect that does what ``effect`` does, but retrying on
exception.
"""
end_time = time() + timeout
def should_retry(e):
if time() >= end_time:
return Effect(Constant(False))
else:
retry_delay = should_retry.wait_secs.total_seconds()
effect = Effect(Delay(retry_delay)).on(
success=lambda x: Effect(Constant(True))
)
if backoff:
should_retry.wait_secs *= 2
return effect
should_retry.wait_secs = retry_wait
return retry(effect, should_retry)