def test_terminal_errors_except_continues_on_provided_exceptions(self):
"""
If the failure is of a type provided to :func:`transient_errors_except`,
the function it returns will treat it as transient (returns True)
"""
can_retry = terminal_errors_except(DummyException)
self.assertTrue(can_retry(Failure(DummyException())))
def wait_for_state(self, rcs, matcher, timeout=600, period=10, clock=None):
"""
Wait for the state on the scaling group to match the provided matchers,
specified by matcher.
:param rcs: a :class:`otter.integration.lib.resources.TestResources`
instance
:param matcher: A :mod:`testtool.matcher`, as specified in
module: testtools.matchers in
http://testtools.readthedocs.org/en/latest/api.html.
:param timeout: The amount of time to wait until this step is
considered failed.
:param period: How long to wait before polling again.
:param clock: a :class:`twisted.internet.interfaces.IReactorTime`
provider
:return: None, if the state is reached
:raises: :class:`TimedOutError` if the state is never reached within
the requisite amount of time.
Example usage:
```
matcher = MatchesAll(
IncludesServers(included_server_ids),
ExcludesServers(exclude_server_ids),
ContainsDict({
'pending': Equals(0),
'desired': Equals(5),
'status': Equals('ACTIVE')
})
)
..wait_for_state(rcs, matchers, timeout=60)
```
"""
def check(result):
response, group_state = result
mismatch = matcher.match(group_state['group'])
if mismatch:
msg("Waiting for group {} to reach desired group state.\n"
"Mismatch: {}"
.format(self.group_id, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
msg("Success: desired group state reached:\n{}\nmatches:\n{}"
.format(group_state['group'], matcher))
return rcs
def poll():
return self.get_scaling_group_state(rcs, [200]).addCallback(check)
return retry_and_timeout(
poll, timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=(
"Waiting for group {} to reach state {}"
.format(self.group_id, str(matcher)))
)
def test_terminal_errors_except_continues_on_provided_exceptions(self):
"""
If the failure is of a type provided to :func:`transient_errors_except`,
the function it returns will treat it as transient (returns True)
"""
can_retry = terminal_errors_except(DummyException)
self.assertTrue(can_retry(Failure(DummyException())))
def wait_for_stack_list(self, expected_states, timeout=180, period=10):
def check(content):
states = pbag([s['stack_status'] for s in content['stacks']])
if not (states == expected_states):
msg("Waiting for group {} to reach desired group state.\n"
"{} (actual) {} (expected)"
.format(self.group.group_id, states, expected_states))
raise TransientRetryError(
"Group states of {} did not match expected {})"
.format(states, expected_states))
msg("Success: desired group state reached:\n{}"
.format(expected_states))
return self.rcs
def poll():
return self.get_stack_list().addCallback(check)
expected_states = pbag(expected_states)
return retry_and_timeout(
poll, timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=reactor,
deferred_description=(
"Waiting for group {} to reach state {}".format(
self.group.group_id, str(expected_states))))
def wait_for_stack_list(self, expected_states, timeout=180, period=10):
def check(content):
states = pbag([s['stack_status'] for s in content['stacks']])
if not (states == expected_states):
msg("Waiting for group {} to reach desired group state.\n"
"{} (actual) {} (expected)".format(self.group.group_id,
states,
expected_states))
raise TransientRetryError(
"Group states of {} did not match expected {})".format(
states, expected_states))
msg("Success: desired group state reached:\n{}".format(
expected_states))
return self.rcs
def poll():
return self.get_stack_list().addCallback(check)
expected_states = pbag(expected_states)
return retry_and_timeout(
poll,
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=reactor,
deferred_description=(
"Waiting for group {} to reach state {}".format(
self.group.group_id, str(expected_states))))
def wait_for_state(self, rcs, matcher, timeout=600, period=10, clock=None):
"""
Wait for the state on the scaling group to match the provided matchers,
specified by matcher.
:param rcs: a :class:`otter.integration.lib.resources.TestResources`
instance
:param matcher: A :mod:`testtool.matcher`, as specified in
module: testtools.matchers in
http://testtools.readthedocs.org/en/latest/api.html.
:param timeout: The amount of time to wait until this step is
considered failed.
:param period: How long to wait before polling again.
:param clock: a :class:`twisted.internet.interfaces.IReactorTime`
provider
:return: None, if the state is reached
:raises: :class:`TimedOutError` if the state is never reached within
the requisite amount of time.
Example usage:
```
matcher = MatchesAll(
IncludesServers(included_server_ids),
ExcludesServers(exclude_server_ids),
ContainsDict({
'pending': Equals(0),
'desired': Equals(5),
'status': Equals('ACTIVE')
})
)
..wait_for_state(rcs, matchers, timeout=60)
```
"""
def check(result):
response, group_state = result
mismatch = matcher.match(group_state['group'])
if mismatch:
msg("Waiting for group {} to reach desired group state.\n"
"Mismatch: {}".format(self.group_id, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
msg("Success: desired group state reached:\n{}\nmatches:\n{}".
format(group_state['group'], matcher))
return rcs
def poll():
return self.get_scaling_group_state(rcs, [200]).addCallback(check)
return retry_and_timeout(
poll,
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=(
"Waiting for group {} to reach state {}".format(
self.group_id, str(matcher))))
def retrier(*args, **kwargs):
return retry_and_timeout(
partial(f, *args, **kwargs), timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock,
deferred_description=reason
)
def retrier(*args, **kwargs):
return retry_and_timeout(
partial(f, *args, **kwargs),
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock,
deferred_description=reason)
def test_terminal_errors_except_defaults_to_all_errors_bad(self):
"""
If no args are provided to :func:`fail_unless`, the
function it returns treats all errors as terminal (returns False)
"""
can_retry = terminal_errors_except()
for exception in (DummyException(), NotImplementedError()):
self.assertFalse(can_retry(Failure(exception)))
def test_terminal_errors_except_defaults_to_all_errors_bad(self):
"""
If no args are provided to :func:`fail_unless`, the
function it returns treats all errors as terminal (returns False)
"""
can_retry = terminal_errors_except()
for exception in (DummyException(), NotImplementedError()):
self.assertFalse(can_retry(Failure(exception)))
def wait_for_servers(rcs,
pool,
matcher,
group=None,
timeout=600,
period=10,
clock=None,
_treq=treq):
"""
Wait until Nova reaches a particular state (as described by the given
matcher) - if a group is provided, then match only the servers for the
given group.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
:param pool: a :class:`twisted.web.client.HTTPConnectionPool`
:param matcher: a :mod:`testtools.matcher` matcher that describes the
desired state of the servers belonging to the autoscaling group.
:param group: a :class:`otter.integration.lib.autoscale.ScalingGroup` that
specifies which autoscaling group's servers we are looking at. This
group should already exist, and have a `group_id` attribute. If not
provided, the matcher will apply to all servers.
"""
message = "Waiting for {0} Nova servers".format(
"all" if group is None else "group {0} 's".format(group.group_id))
@inlineCallbacks
def do_work():
servers = yield list_servers(rcs, pool, _treq=_treq)
servers = servers['servers']
if group is not None:
servers = [
server for server in servers
if (group.group_id == server['metadata'].get(
"rax:autoscale:group:id", None))
]
mismatch = matcher.match(servers)
if mismatch:
msg("{0}.\nMismatch: {1}".format(message, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
returnValue(servers)
return retry_and_timeout(
do_work,
timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=("{0} to reach state {1}".format(
message, str(matcher))))
def wait_for_servers(rcs, pool, matcher, group=None, timeout=600, period=10,
clock=None, _treq=treq):
"""
Wait until Nova reaches a particular state (as described by the given
matcher) - if a group is provided, then match only the servers for the
given group.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
:param pool: a :class:`twisted.web.client.HTTPConnectionPool`
:param matcher: a :mod:`testtools.matcher` matcher that describes the
desired state of the servers belonging to the autoscaling group.
:param group: a :class:`otter.integration.lib.autoscale.ScalingGroup` that
specifies which autoscaling group's servers we are looking at. This
group should already exist, and have a `group_id` attribute. If not
provided, the matcher will apply to all servers.
"""
message = "Waiting for {0} Nova servers".format(
"all" if group is None else "group {0} 's".format(group.group_id))
@inlineCallbacks
def do_work():
servers = yield list_servers(rcs, pool, _treq=_treq)
servers = servers['servers']
if group is not None:
servers = [
server for server in servers
if (group.group_id ==
server['metadata'].get("rax:autoscale:group:id", None))
]
mismatch = matcher.match(servers)
if mismatch:
msg("{0}.\nMismatch: {1}".format(message, mismatch.describe()))
raise TransientRetryError(mismatch.describe())
returnValue(servers)
return retry_and_timeout(
do_work, timeout,
can_retry=terminal_errors_except(TransientRetryError),
next_interval=repeating_interval(period),
clock=clock or reactor,
deferred_description=(
"{0} to reach state {1}".format(message, str(matcher)))
)
def delete(self, rcs):
"""
Delete the server.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
"""
def try_delete():
d = self.treq.delete(
"{}/servers/{}".format(rcs.endpoints["nova"], self.id),
headers=headers(str(rcs.token)),
pool=self.pool)
d.addCallback(check_success, [404], _treq=self.treq)
d.addCallback(self.treq.content)
return d
return retry_and_timeout(
try_delete, 120,
can_retry=terminal_errors_except(APIError),
next_interval=repeating_interval(5),
clock=self.clock,
deferred_description=(
"Waiting for server {} to get deleted".format(self.id)))
def delete(self, rcs):
"""
Delete the server.
:param rcs: an instance of
:class:`otter.integration.lib.resources.TestResources`
"""
def try_delete():
d = self.treq.delete("{}/servers/{}".format(
rcs.endpoints["nova"], self.id),
headers=headers(str(rcs.token)),
pool=self.pool)
d.addCallback(check_success, [404], _treq=self.treq)
d.addCallback(self.treq.content)
return d
return retry_and_timeout(
try_delete,
120,
can_retry=terminal_errors_except(APIError),
next_interval=repeating_interval(5),
clock=self.clock,
deferred_description=(
"Waiting for server {} to get deleted".format(self.id)))
def create_server(server_endpoint,
auth_token,
server_config,
log=None,
clock=None,
retries=3,
create_failure_delay=5,
_treq=None):
"""
Create a new server. If there is an error from Nova from this call,
checks to see if the server was created anyway. If not, will retry the
create ``retries`` times (checking each time if a server).
If the error from Nova is a 400, does not retry, because that implies that
retrying will just result in another 400 (bad args).
If checking to see if the server is created also results in a failure,
does not retry because there might just be something wrong with Nova.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth Token.
:param dict server_config: Nova server config.
:param: int retries: Number of tries to retry the create.
:param: int create_failure_delay: how much time in seconds to wait after
a create server failure before checking Nova to see if a server
was created
:param log: logger
:type log: :class:`otter.log.bound.BoundLog`
:param _treq: To be used for testing - what treq object to use
:type treq: something with the same api as :obj:`treq`
:return: Deferred that fires with the CreateServer response as a dict.
"""
path = append_segments(server_endpoint, 'servers')
if _treq is None: # pragma: no cover
_treq = treq
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
def _check_results(result, propagated_f):
"""
Return the original failure, if checking a server resulted in a
failure too. Returns a wrapped propagated failure, if there were no
servers created, so that the retry utility knows that server creation
can be retried.
"""
if isinstance(result, Failure):
log.msg(
"Attempt to find a created server in nova resulted in "
"{failure}. Propagating the original create error instead.",
failure=result)
return propagated_f
if result is None:
raise _NoCreatedServerFound(propagated_f)
return result
def _check_server_created(f):
"""
If creating a server failed with anything other than a 400, see if
Nova created a server anyway (a 400 means that the server creation args
were bad, and there is no point in retrying).
If Nova created a server, just return it and pretend that the error
never happened. If it didn't, or if checking resulted in another
failure response, return a failure of some type.
"""
f.trap(APIError)
if f.value.code == 400:
return f
d = deferLater(clock,
create_failure_delay,
find_server,
server_endpoint,
auth_token,
server_config,
log=log)
d.addBoth(_check_results, f)
return d
def _create_with_delay(to_delay):
d = _treq.post(path,
headers=headers(auth_token),
data=json.dumps({'server': server_config}),
log=log)
if to_delay:
# Add 1 second delay to space 1 second between server creations
d.addCallback(delay, clock, 1)
return d
def _create_server():
"""
Attempt to create a server, handling spurious non-400 errors from Nova
by seeing if Nova created a server anyway in spite of the error. If so
then create server succeeded.
If not, and if no further errors occur, server creation can be retried.
"""
sem = get_sempahore("create_server", "worker.create_server_limit")
if sem is not None:
d = sem.run(_create_with_delay, True)
else:
d = _create_with_delay(False)
d.addCallback(check_success, [202], _treq=_treq)
d.addCallback(_treq.json_content)
d.addErrback(_check_server_created)
return d
def _unwrap_NoCreatedServerFound(f):
"""
The original failure was wrapped in a :class:`_NoCreatedServerFound`
for ease of retry, but that should not be the final error propagated up
by :func:`create_server`.
This errback unwraps the :class:`_NoCreatedServerFound` error and
returns the original failure.
"""
f.trap(_NoCreatedServerFound)
return f.value.original
d = retry(_create_server,
can_retry=compose_retries(
retry_times(retries),
terminal_errors_except(_NoCreatedServerFound)),
next_interval=repeating_interval(15),
clock=clock)
d.addErrback(_unwrap_NoCreatedServerFound)
d.addErrback(wrap_request_error, path, 'server_create')
return d
def create_server(server_endpoint, auth_token, server_config, log=None,
clock=None, retries=3, create_failure_delay=5, _treq=None):
"""
Create a new server. If there is an error from Nova from this call,
checks to see if the server was created anyway. If not, will retry the
create ``retries`` times (checking each time if a server).
If the error from Nova is a 400, does not retry, because that implies that
retrying will just result in another 400 (bad args).
If checking to see if the server is created also results in a failure,
does not retry because there might just be something wrong with Nova.
:param str server_endpoint: Server endpoint URI.
:param str auth_token: Keystone Auth Token.
:param dict server_config: Nova server config.
:param: int retries: Number of tries to retry the create.
:param: int create_failure_delay: how much time in seconds to wait after
a create server failure before checking Nova to see if a server
was created
:param log: logger
:type log: :class:`otter.log.bound.BoundLog`
:param _treq: To be used for testing - what treq object to use
:type treq: something with the same api as :obj:`treq`
:return: Deferred that fires with the CreateServer response as a dict.
"""
path = append_segments(server_endpoint, 'servers')
if _treq is None: # pragma: no cover
_treq = treq
if clock is None: # pragma: no cover
from twisted.internet import reactor
clock = reactor
def _check_results(result, propagated_f):
"""
Return the original failure, if checking a server resulted in a
failure too. Returns a wrapped propagated failure, if there were no
servers created, so that the retry utility knows that server creation
can be retried.
"""
if isinstance(result, Failure):
log.msg("Attempt to find a created server in nova resulted in "
"{failure}. Propagating the original create error instead.",
failure=result)
return propagated_f
if result is None:
raise _NoCreatedServerFound(propagated_f)
return result
def _check_server_created(f):
"""
If creating a server failed with anything other than a 400, see if
Nova created a server anyway (a 400 means that the server creation args
were bad, and there is no point in retrying).
If Nova created a server, just return it and pretend that the error
never happened. If it didn't, or if checking resulted in another
failure response, return a failure of some type.
"""
f.trap(APIError)
if f.value.code == 400:
return f
d = deferLater(clock, create_failure_delay, find_server,
server_endpoint, auth_token, server_config, log=log)
d.addBoth(_check_results, f)
return d
def _create_with_delay(to_delay):
d = _treq.post(path, headers=headers(auth_token),
data=json.dumps({'server': server_config}), log=log)
if to_delay:
# Add 1 second delay to space 1 second between server creations
d.addCallback(delay, clock, 1)
return d
def _create_server():
"""
Attempt to create a server, handling spurious non-400 errors from Nova
by seeing if Nova created a server anyway in spite of the error. If so
then create server succeeded.
If not, and if no further errors occur, server creation can be retried.
"""
sem = get_sempahore("create_server", "worker.create_server_limit")
if sem is not None:
d = sem.run(_create_with_delay, True)
else:
d = _create_with_delay(False)
d.addCallback(check_success, [202], _treq=_treq)
d.addCallback(_treq.json_content)
d.addErrback(_check_server_created)
return d
def _unwrap_NoCreatedServerFound(f):
"""
The original failure was wrapped in a :class:`_NoCreatedServerFound`
for ease of retry, but that should not be the final error propagated up
by :func:`create_server`.
This errback unwraps the :class:`_NoCreatedServerFound` error and
returns the original failure.
"""
f.trap(_NoCreatedServerFound)
return f.value.original
d = retry(
_create_server,
can_retry=compose_retries(
retry_times(retries),
terminal_errors_except(_NoCreatedServerFound)),
next_interval=repeating_interval(15), clock=clock)
d.addErrback(_unwrap_NoCreatedServerFound)
d.addErrback(wrap_request_error, path, 'server_create')
return d
