def test_retry_and_timeout_get_the_same_default_clock(self):
"""
If no clock is passed to ``retry_and_timeout``, both ``retry`` and
``timeout`` nevertheless get the same clock.
"""
retry_and_timeout('do_work', 'timeout')
retry_clock = self.retry.call_args[1]['clock']
timeout_clock = self.timeout.call_args[1]['clock']
self.assertIs(retry_clock, timeout_clock)
def test_both_called_with_all_args(self):
"""
Both ``retry`` and ``timeout`` gets called with the args passed to
``retry_and_timeout``, including the same clock
"""
clock = mock.MagicMock()
retry_and_timeout('do_work', 'timeout', can_retry='can_retry',
next_interval='next_interval', clock=clock)
self.retry.assert_called_once_with('do_work', can_retry='can_retry',
next_interval='next_interval',
clock=clock)
self.timeout.assert_called_once_with(self.retry.return_value,
'timeout', clock=clock)
def verify(_):
def check_status():
check_d = treq.head(
append_segments(server_endpoint, 'servers', server_id),
headers=headers(auth_token))
check_d.addCallback(check_success, [404])
return check_d
start_time = clock.seconds()
# this is treating all errors as transient, so the only error that can
# occur is a CancelledError from timing out
verify_d = retry_and_timeout(check_status, timeout,
next_interval=repeating_interval(interval),
clock=clock)
def on_success(_):
time_delete = clock.seconds() - start_time
del_log.msg('Server deleted successfully: {time_delete} seconds.',
time_delete=time_delete)
verify_d.addCallback(on_success)
def on_timeout(_):
time_delete = clock.seconds() - start_time
del_log.err(None, timeout=timeout, time_delete=time_delete,
why=('Server {instance_id} failed to be deleted within '
'a {timeout} second timeout (it has been '
'{time_delete} seconds).'))
verify_d.addErrback(on_timeout)
def wait_for_stack_list(self, expected_states, timeout=180, period=10):
def check(content):
states = pbag([s['stack_status'] for s in content['stacks']])
if not (states == expected_states):
msg("Waiting for group {} to reach desired group state.\n"
"{} (actual) {} (expected)".format(self.group.group_id,
states,
expected_states))
raise TransientRetryError(
"Group states of {} did not match expected {})".format(
states, expected_states))
msg("Success: desired group state reached:\n{}".format(
expected_states))
return self.rcs
def poll():
return self.get_stack_list().addCallback(check)
expected_states = pbag(expected_states)
return retry_and_timeout(
poll,
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=reactor,
deferred_description=(
"Waiting for group {} to reach state {}".format(
self.group.group_id, str(expected_states))))
def wait_for_stack_list(self, expected_states, timeout=180, period=10):
def check(content):
states = pbag([s['stack_status'] for s in content['stacks']])
if not (states == expected_states):
msg("Waiting for group {} to reach desired group state.\n"
"{} (actual) {} (expected)"
.format(self.group.group_id, states, expected_states))
raise TransientRetryError(
"Group states of {} did not match expected {})"
.format(states, expected_states))
msg("Success: desired group state reached:\n{}"
.format(expected_states))
return self.rcs
def poll():
return self.get_stack_list().addCallback(check)
expected_states = pbag(expected_states)
return retry_and_timeout(
poll, timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=reactor,
deferred_description=(
"Waiting for group {} to reach state {}".format(
self.group.group_id, str(expected_states))))
def verify(_):
def check_status():
check_d = treq.head(
append_segments(server_endpoint, 'servers', server_id),
headers=headers(auth_token))
check_d.addCallback(check_success, [404])
return check_d
start_time = clock.seconds()
timeout_description = (
"Waiting for Nova to actually delete server {0}".format(server_id))
verify_d = retry_and_timeout(check_status, timeout,
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
def on_success(_):
time_delete = clock.seconds() - start_time
del_log.msg('Server deleted successfully: {time_delete} seconds.',
time_delete=time_delete)
verify_d.addCallback(on_success)
verify_d.addErrback(del_log.err)
def wait_for_state(self, rcs, matcher, timeout=600, period=10, clock=None):
"""
Wait for the state on the scaling group to match the provided matchers,
specified by matcher.
:param rcs: a :class:`otter.integration.lib.resources.TestResources`
instance
:param matcher: A :mod:`testtool.matcher`, as specified in
module: testtools.matchers in
http://testtools.readthedocs.org/en/latest/api.html.
:param timeout: The amount of time to wait until this step is
considered failed.
:param period: How long to wait before polling again.
:param clock: a :class:`twisted.internet.interfaces.IReactorTime`
provider
:return: None, if the state is reached
:raises: :class:`TimedOutError` if the state is never reached within
the requisite amount of time.
Example usage:
```
matcher = MatchesAll(
IncludesServers(included_server_ids),
ExcludesServers(exclude_server_ids),
ContainsDict({
'pending': Equals(0),
'desired': Equals(5),
'status': Equals('ACTIVE')
})
)
..wait_for_state(rcs, matchers, timeout=60)
```
"""
def check(result):
response, group_state = result
mismatch = matcher.match(group_state['group'])
if mismatch:
msg("Waiting for group {} to reach desired group state.\n"
"Mismatch: {}"
.format(self.group_id, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
msg("Success: desired group state reached:\n{}\nmatches:\n{}"
.format(group_state['group'], matcher))
return rcs
def poll():
return self.get_scaling_group_state(rcs, [200]).addCallback(check)
return retry_and_timeout(
poll, timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=(
"Waiting for group {} to reach state {}"
.format(self.group_id, str(matcher)))
)
def wait_for_state(self, rcs, matcher, timeout=600, period=10, clock=None):
"""
Wait for the state on the scaling group to match the provided matchers,
specified by matcher.
:param rcs: a :class:`otter.integration.lib.resources.TestResources`
instance
:param matcher: A :mod:`testtool.matcher`, as specified in
module: testtools.matchers in
http://testtools.readthedocs.org/en/latest/api.html.
:param timeout: The amount of time to wait until this step is
considered failed.
:param period: How long to wait before polling again.
:param clock: a :class:`twisted.internet.interfaces.IReactorTime`
provider
:return: None, if the state is reached
:raises: :class:`TimedOutError` if the state is never reached within
the requisite amount of time.
Example usage:
```
matcher = MatchesAll(
IncludesServers(included_server_ids),
ExcludesServers(exclude_server_ids),
ContainsDict({
'pending': Equals(0),
'desired': Equals(5),
'status': Equals('ACTIVE')
})
)
..wait_for_state(rcs, matchers, timeout=60)
```
"""
def check(result):
response, group_state = result
mismatch = matcher.match(group_state['group'])
if mismatch:
msg("Waiting for group {} to reach desired group state.\n"
"Mismatch: {}".format(self.group_id, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
msg("Success: desired group state reached:\n{}\nmatches:\n{}".
format(group_state['group'], matcher))
return rcs
def poll():
return self.get_scaling_group_state(rcs, [200]).addCallback(check)
return retry_and_timeout(
poll,
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=(
"Waiting for group {} to reach state {}".format(
self.group_id, str(matcher))))
def retrier(*args, **kwargs):
return retry_and_timeout(
partial(f, *args, **kwargs),
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock,
deferred_description=reason)
def retrier(*args, **kwargs):
return retry_and_timeout(
partial(f, *args, **kwargs), timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock,
deferred_description=reason
)
def verified_delete(log,
server_endpoint,
auth_token,
server_id,
interval=10,
timeout=3660,
clock=None):
"""
Attempt to delete a server from the server endpoint, and ensure that it is
deleted by trying again until deleting/getting the server results in a 404
or until ``OS-EXT-STS:task_state`` in server details is 'deleting',
indicating that Nova has acknowledged that the server is to be deleted
as soon as possible.
Time out attempting to verify deletes after a period of time and log an
error.
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Deletion interval in seconds - how long until
verifying a delete is retried. Default: 5.
:param int timeout: Seconds after which the deletion will be logged as a
failure, if Nova fails to return a 404. Default is 3660, because if
the server is building, the delete will not happen until immediately
after it has finished building.
:return: Deferred that fires when the expected status has been seen.
"""
serv_log = log.bind(server_id=server_id)
serv_log.msg('Deleting server')
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
timeout_description = (
"Waiting for Nova to actually delete server {0} (or acknowledge delete)"
.format(server_id))
d = retry_and_timeout(
partial(delete_and_verify, serv_log, server_endpoint, auth_token, server_id),
timeout,
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
d.addCallback(log_with_time, clock, serv_log, clock.seconds(),
('Server deleted successfully (or acknowledged by Nova as '
'to-be-deleted) : {time_delete} seconds.'), 'time_delete')
d.addErrback(serv_log.err)
return d
def wait_for_servers(rcs,
pool,
matcher,
group=None,
timeout=600,
period=10,
clock=None,
_treq=treq):
"""
Wait until Nova reaches a particular state (as described by the given
matcher) - if a group is provided, then match only the servers for the
given group.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
:param pool: a :class:`twisted.web.client.HTTPConnectionPool`
:param matcher: a :mod:`testtools.matcher` matcher that describes the
desired state of the servers belonging to the autoscaling group.
:param group: a :class:`otter.integration.lib.autoscale.ScalingGroup` that
specifies which autoscaling group's servers we are looking at. This
group should already exist, and have a `group_id` attribute. If not
provided, the matcher will apply to all servers.
"""
message = "Waiting for {0} Nova servers".format(
"all" if group is None else "group {0} 's".format(group.group_id))
@inlineCallbacks
def do_work():
servers = yield list_servers(rcs, pool, _treq=_treq)
servers = servers['servers']
if group is not None:
servers = [
server for server in servers
if (group.group_id == server['metadata'].get(
"rax:autoscale:group:id", None))
]
mismatch = matcher.match(servers)
if mismatch:
msg("{0}.\nMismatch: {1}".format(message, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
returnValue(servers)
return retry_and_timeout(
do_work,
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=("{0} to reach state {1}".format(
message, str(matcher))))
def wait_for_servers(rcs, pool, matcher, group=None, timeout=600, period=10,
clock=None, _treq=treq):
"""
Wait until Nova reaches a particular state (as described by the given
matcher) - if a group is provided, then match only the servers for the
given group.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
:param pool: a :class:`twisted.web.client.HTTPConnectionPool`
:param matcher: a :mod:`testtools.matcher` matcher that describes the
desired state of the servers belonging to the autoscaling group.
:param group: a :class:`otter.integration.lib.autoscale.ScalingGroup` that
specifies which autoscaling group's servers we are looking at. This
group should already exist, and have a `group_id` attribute. If not
provided, the matcher will apply to all servers.
"""
message = "Waiting for {0} Nova servers".format(
"all" if group is None else "group {0} 's".format(group.group_id))
@inlineCallbacks
def do_work():
servers = yield list_servers(rcs, pool, _treq=_treq)
servers = servers['servers']
if group is not None:
servers = [
server for server in servers
if (group.group_id ==
server['metadata'].get("rax:autoscale:group:id", None))
]
mismatch = matcher.match(servers)
if mismatch:
msg("{0}.\nMismatch: {1}".format(message, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
returnValue(servers)
return retry_and_timeout(
do_work, timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=(
"{0} to reach state {1}".format(message, str(matcher)))
)
def delete(self, rcs):
"""
Delete the server.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
"""
def try_delete():
d = self.treq.delete(
"{}/servers/{}".format(rcs.endpoints["nova"], self.id),
headers=headers(str(rcs.token)),
pool=self.pool)
d.addCallback(check_success, [404], _treq=self.treq)
d.addCallback(self.treq.content)
return d
return retry_and_timeout(
try_delete, 120,
can_retry=terminal_errors_except(APIError),
next_interval=repeating_interval(5),
clock=self.clock,
deferred_description=(
"Waiting for server {} to get deleted".format(self.id)))
def verify(_):
def check_status():
check_d = treq.head(append_segments(server_endpoint, 'servers',
server_id),
headers=headers(auth_token))
check_d.addCallback(check_success, [404])
return check_d
start_time = clock.seconds()
# this is treating all errors as transient, so the only error that can
# occur is a CancelledError from timing out
verify_d = retry_and_timeout(
check_status,
timeout,
next_interval=repeating_interval(interval),
clock=clock)
def on_success(_):
time_delete = clock.seconds() - start_time
del_log.msg('Server deleted successfully: {time_delete} seconds.',
time_delete=time_delete)
verify_d.addCallback(on_success)
def on_timeout(_):
time_delete = clock.seconds() - start_time
del_log.err(
None,
timeout=timeout,
time_delete=time_delete,
why=('Server {instance_id} failed to be deleted within '
'a {timeout} second timeout (it has been '
'{time_delete} seconds).'))
verify_d.addErrback(on_timeout)
def delete(self, rcs):
"""
Delete the server.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
"""
def try_delete():
d = self.treq.delete("{}/servers/{}".format(
rcs.endpoints["nova"], self.id),
headers=headers(str(rcs.token)),
pool=self.pool)
d.addCallback(check_success, [404], _treq=self.treq)
d.addCallback(self.treq.content)
return d
return retry_and_timeout(
try_delete,
120,
can_retry=terminal_errors_except(APIError),
next_interval=repeating_interval(5),
clock=self.clock,
deferred_description=(
"Waiting for server {} to get deleted".format(self.id)))
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=20,
timeout=7200,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 20.
:param int timeout: timeout to poll for the server status in seconds.
Default 7200 (2 hours).
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
time_building = clock.seconds() - start_time
if status == 'ACTIVE':
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
log.msg(
"Server changed to '{status}' in {time_building} seconds",
time_building=time_building,
status=status)
raise UnexpectedServerStatus(server_id, status, 'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id, log=log)
sd.addCallback(check_status)
return sd
timeout_description = ("Waiting for server <{0}> to change from BUILD "
"state to ACTIVE state").format(server_id)
return retry_and_timeout(poll,
timeout,
can_retry=transient_errors_except(
UnexpectedServerStatus, ServerDeleted),
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=20,
timeout=7200,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 20.
:param int timeout: timeout to poll for the server status in seconds.
Default 7200 (2 hours).
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
time_building = clock.seconds() - start_time
if status == 'ACTIVE':
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
log.msg("Server changed to '{status}' in {time_building} seconds",
time_building=time_building, status=status)
raise UnexpectedServerStatus(
server_id,
status,
'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id, log=log)
sd.addCallback(check_status)
return sd
timeout_description = ("Waiting for server <{0}> to change from BUILD "
"state to ACTIVE state").format(server_id)
return retry_and_timeout(
poll, timeout,
can_retry=transient_errors_except(UnexpectedServerStatus, ServerDeleted),
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
def verified_delete(log,
server_endpoint,
auth_token,
server_id,
interval=10,
timeout=3660,
clock=None):
"""
Attempt to delete a server from the server endpoint, and ensure that it is
deleted by trying again until deleting the server results in a 404.
Time out attempting to verify deletes after a period of time and log an
error.
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Deletion interval in seconds - how long until
verifying a delete is retried. Default: 5.
:param int timeout: Seconds after which the deletion will be logged as a
failure, if Nova fails to return a 404. Default is 3660, because if
the server is building, the delete will not happen until immediately
after it has finished building.
:return: Deferred that fires when the expected status has been seen.
"""
serv_log = log.bind(server_id=server_id)
serv_log.msg('Deleting server')
path = append_segments(server_endpoint, 'servers', server_id)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
# just delete over and over until a 404 is received
def delete():
del_d = treq.delete(path, headers=headers(auth_token), log=serv_log)
del_d.addCallback(check_success, [404])
del_d.addCallback(treq.content)
return del_d
start_time = clock.seconds()
timeout_description = (
"Waiting for Nova to actually delete server {0}".format(server_id))
d = retry_and_timeout(delete, timeout,
next_interval=repeating_interval(interval),
clock=clock,
deferred_description=timeout_description)
def on_success(_):
time_delete = clock.seconds() - start_time
serv_log.msg('Server deleted successfully: {time_delete} seconds.',
time_delete=time_delete)
d.addCallback(on_success)
d.addErrback(serv_log.err)
return d
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=5,
timeout=3600,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 5.
:param int timeout: timeout to poll for the server status in seconds.
Default 3600 (1 hour)
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
if status == 'ACTIVE':
time_building = clock.seconds() - start_time
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
raise UnexpectedServerStatus(server_id, status, 'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id)
sd.addCallback(check_status)
return sd
d = retry_and_timeout(
poll,
timeout,
can_retry=transient_errors_except(UnexpectedServerStatus),
next_interval=repeating_interval(interval),
clock=clock)
def on_error(f):
if f.check(CancelledError):
time_building = clock.seconds() - start_time
log.msg(
('Server {instance_id} failed to change from BUILD state '
'to ACTIVE within a {timeout} second timeout (it has been '
'{time_building} seconds).'),
timeout=timeout,
time_building=time_building)
return f
d.addErrback(on_error)
return d
def wait_for_active(log,
server_endpoint,
auth_token,
server_id,
interval=5,
timeout=3600,
clock=None):
"""
Wait until the server specified by server_id's status is 'ACTIVE'
:param log: A bound logger.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth token.
:param str server_id: Opaque nova server id.
:param int interval: Polling interval in seconds. Default: 5.
:param int timeout: timeout to poll for the server status in seconds.
Default 3600 (1 hour)
:return: Deferred that fires when the expected status has been seen.
"""
log.msg("Checking instance status every {interval} seconds",
interval=interval)
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
start_time = clock.seconds()
def poll():
def check_status(server):
status = server['server']['status']
if status == 'ACTIVE':
time_building = clock.seconds() - start_time
log.msg(("Server changed from 'BUILD' to 'ACTIVE' within "
"{time_building} seconds"),
time_building=time_building)
return server
elif status != 'BUILD':
raise UnexpectedServerStatus(
server_id,
status,
'ACTIVE')
else:
raise TransientRetryError() # just poll again
sd = server_details(server_endpoint, auth_token, server_id)
sd.addCallback(check_status)
return sd
d = retry_and_timeout(
poll, timeout,
can_retry=transient_errors_except(UnexpectedServerStatus),
next_interval=repeating_interval(interval),
clock=clock)
def on_error(f):
if f.check(CancelledError):
time_building = clock.seconds() - start_time
log.msg(('Server {instance_id} failed to change from BUILD state '
'to ACTIVE within a {timeout} second timeout (it has been '
'{time_building} seconds).'),
timeout=timeout, time_building=time_building)
return f
d.addErrback(on_error)
return d