def reporter(exc_tuple):
err_type, error, traceback = exc_tuple
terminal_error = (any(
issubclass(err_type, etype) for etype in terminal_err_types)
or err_type == APIError and 400 <= error.code < 500)
if terminal_error:
return StepResult.FAILURE, [ErrorReason.Exception(exc_tuple)]
return StepResult.RETRY, [ErrorReason.Exception(exc_tuple)]
def _handle_bulk_add_errors(exc_tuple):
error = exc_tuple[1]
failures = []
retries = []
for excp in error.errors:
if isinstance(excp, rcv3.ServerUnprocessableError):
retries.append(ErrorReason.String(excp.message))
else:
failures.append(ErrorReason.String(excp.message))
if failures:
return StepResult.FAILURE, failures
else:
return StepResult.RETRY, retries
def test_failures(self):
"""
If `rcv3.bulk_add` results in BulkErrors with only
non-ServerUnprocessableError errors in it then step returns FAILURE
"""
excp1 = rcv3.LBInactive("l1")
excp2 = rcv3.NoSuchLBError("l2")
seq = [(("ba", self.pairs), self.ba_raiser(excp1, excp2))]
self.assertEqual(
perform_sequence(seq, self.step.as_effect()),
(StepResult.FAILURE,
transform_eq(pset, pset([
ErrorReason.String(excp1.message),
ErrorReason.String(excp2.message)])))
)
def test_retries(self):
"""
If `rcv3.bulk_add` results in BulkErrors with only
ServerUnprocessableError errors in it then step returns RETRY
"""
excp1 = rcv3.ServerUnprocessableError("s1")
excp2 = rcv3.ServerUnprocessableError("s2")
seq = [(("ba", self.pairs), self.ba_raiser(excp1, excp2))]
self.assertEqual(
perform_sequence(seq, self.step.as_effect()),
(StepResult.RETRY,
transform_eq(pset, pset([
ErrorReason.String(excp1.message),
ErrorReason.String(excp2.message)])))
)
def test_present_exceptions(self):
"""Some exceptions are presented."""
excs = {
NoSuchCLBError(lb_id=u'lbid1'):
'Cloud Load Balancer does not exist: lbid1',
CLBDeletedError(lb_id=u'lbid2'):
'Cloud Load Balancer is currently being deleted: lbid2',
NoSuchCLBNodeError(lb_id=u'lbid3', node_id=u'node1'):
"Node node1 of Cloud Load Balancer lbid3 does not exist",
CLBNodeLimitError(lb_id=u'lb2', node_limit=25):
"Cannot create more than 25 nodes in Cloud Load Balancer lb2",
CLBHealthInfoNotFound(u'lb2'):
"Could not find health monitor configuration of "
"Cloud Load Balancer lb2",
CreateServerConfigurationError("Your server is wrong"):
'Server launch configuration is invalid: Your server is wrong',
CreateServerOverQuoteError("You are over quota"):
'Servers cannot be created: You are over quota',
NoSuchEndpoint(service_name="nova", region="ord"):
"Could not locate service nova in the service catalog. "
"Please check if your account is still active."
}
excs = excs.items()
self.assertEqual(
present_reasons([
ErrorReason.Exception(raise_to_exc_info(exc))
for (exc, _) in excs
]), [reason for (_, reason) in excs])
def test_good_response(self):
"""
If the response code indicates success, the step returns a RETRY so
that another convergence cycle can be done to update the active server
list.
"""
node_a_id = '825b8c72-9951-4aff-9cd8-fa3ca5551c90'
lb_a_id = '2b0e17b6-0429-4056-b86c-e670ad5de853'
node_b_id = "d6d3aa7c-dfa5-4e61-96ee-1d54ac1075d2"
lb_b_id = 'd95ae0c4-6ab8-4873-b82f-f8433840cff2'
pairs = [(lb_a_id, node_a_id), (lb_b_id, node_b_id)]
resp = StubResponse(201, {})
body = [{"cloud_server": {"id": node_id},
"load_balancer_pool": {"id": lb_id}}
for (lb_id, node_id) in pairs]
res = _rcv3_check_bulk_add(pairs, (resp, body))
self.assertEqual(
res,
(StepResult.RETRY,
[ErrorReason.String(
'must re-gather after adding to LB in order to update the '
'active cache')]))
def steps_to_effect(steps):
"""Turns a collection of :class:`IStep` providers into an effect."""
# Treat unknown errors as RETRY.
return parallel([
s.as_effect().on(error=lambda e: (StepResult.RETRY,
[ErrorReason.Exception(e)]))
for s in steps])
def change_lb_node(node, description, lb, now, timeout):
"""
Change the configuration of a load balancer node to desired description.
If CLB has health monitor enabled and the node is DRAINING then it will be
ENABLEDed.
:param node: The node to be changed.
:type node: :class:`ILBNode` provider
:param description: The description of the load balancer and how to add
the server to it.
:type description: :class:`ILBDescription` provider
:param float now: Number of seconds since EPOCH
:param float timeout: How long can node remain OFFLINE after adding
in seconds?
:return: :obj:`IStep` object or None
"""
if (type(node.description) == type(description) and
isinstance(description, CLBDescription)):
if lb is None:
return fail_convergence(CLBHealthInfoNotFound(description.lb_id))
if (lb.health_monitor and
node.description.condition == CLBNodeCondition.DRAINING):
# Enable node if it is ONLINE
if node.is_online:
return ChangeCLBNode(lb_id=description.lb_id,
node_id=node.node_id,
condition=CLBNodeCondition.ENABLED,
weight=description.weight,
type=description.type)
# For a new node created in DRAINING, drained_at represents
# node's creation time.
if now - node.drained_at > timeout:
rsfmt = ("Node {} has remained OFFLINE for more than "
"{} seconds")
return FailConvergence(
[ErrorReason.String(rsfmt.format(node.node_id, timeout))])
else:
return ConvergeLater(
[ErrorReason.String(("Waiting for node {} to come "
"ONLINE").format(node.node_id))])
return ChangeCLBNode(lb_id=description.lb_id,
node_id=node.node_id,
condition=description.condition,
weight=description.weight,
type=description.type)
def test_exception(self):
"""Exceptions get serialized along with their traceback."""
exc_info = raise_to_exc_info(ZeroDivisionError('foo'))
reason = ErrorReason.Exception(exc_info)
expected_tb = ''.join(traceback.format_exception(*exc_info))
self.assertEqual(structure_reason(reason), {
'exception': "ZeroDivisionError('foo',)",
'traceback': expected_tb
})
def as_effect(self):
"""Produce a :obj:`Effect` to modify a load balancer node."""
eff = change_clb_node(self.lb_id, self.node_id, weight=self.weight,
condition=self.condition.name,
_type=self.type.name)
return eff.on(
success=lambda _: (StepResult.RETRY, [ErrorReason.String(
'must re-gather after CLB change in order to update the '
'active cache')]),
error=_failure_reporter(CLBNotFoundError, NoSuchCLBNodeError))
def test_ensure_retry(self):
"""Tests that retry will be returned."""
seq = [
(self.check_call.intent, lambda _: (StubResponse(204, ''), None)),
(Log('request-check-stack', ANY), lambda _: None)
]
reason = 'Waiting for stack check to complete'
result = perform_sequence(seq, CheckStack(self.stack).as_effect())
self.assertEqual(result,
(StepResult.RETRY, [ErrorReason.String(reason)]))
def test_retry_default(self):
"""Tests correct behavior when retry is not specified."""
seq = [
(self.update_call.intent, lambda _: (StubResponse(202, ''), None)),
(Log('request-update-stack', ANY), lambda _: None)
]
update = UpdateStack(stack=self.stack, stack_config=self.config)
reason = 'Waiting for stack to update'
result = perform_sequence(seq, update.as_effect())
self.assertEqual(result,
(StepResult.RETRY, [ErrorReason.String(reason)]))
def test_success(self):
"""
Returns RETRY if rcv3.bulk_delete succeeds
"""
seq = [(("bd", self.pairs), noop)]
self.assertEqual(
perform_sequence(seq, self.step.as_effect()),
(StepResult.RETRY, [ErrorReason.String(
'must re-gather after RCv3 LB change in order to update the '
'active cache')])
)
def test_success(self):
"""
A successful return from `rcv3.bulk_add` results in RETRY
"""
seq = [(("ba", self.pairs), noop)]
self.assertEqual(
perform_sequence(seq, self.step.as_effect()),
(StepResult.RETRY, [
ErrorReason.String(
'must re-gather after LB add in order to update the '
'active cache')])
)
def as_effect(self):
"""
Produce a :obj:`Effect` to add some nodes to some RCv3 load
balancers.
"""
eff = rcv3.bulk_add(self.lb_node_pairs)
return eff.on(success=lambda _: (StepResult.RETRY, [
ErrorReason.String(
'must re-gather after LB add in order to update the '
'active cache')
]),
error=catch(rcv3.BulkErrors, _handle_bulk_add_errors))
def _remove_from_lb_with_draining(timeout, nodes, now):
"""
Produce a series of steps that will eventually remove all the given nodes.
It does this in three steps:
For any particular node in ``nodes``:
1. If the timeout is greater than zero, and the node is ``ENABLED``, the
node will be changed to ``DRAINING``.
2. If the node is ``DRAINING``, and the timeout (greater than zero) has
already expired or there are no more active connections, the node will
be removed from the load balancer. If the timeout (greater than zero)
has not expired and active connections != 0, then nothing is done to the
node.
3. If the node is in any other state other than `DRAINING` or `ENABLED`, or
if the timeout is zero, it will be removed from the load balancer.
:param float timeout: the time the node should remain in draining until
removed
:param list nodes: `list` of :obj:`CLBNode` that should be
drained, then removed
:param float now: number of seconds since the POSIX epoch indicating the
time at which the convergence was requested.
:rtype: `list` of :class:`IStep`
"""
to_drain = ()
in_drain = ()
# only put nodes into draining if a timeout is specified
if timeout > 0:
draining, to_drain = partition_bool(
lambda node: node.currently_draining(),
[node for node in nodes
if IDrainable.providedBy(node) and node.is_active()])
# Nothing should be done to these, because the timeout has not expired
# and the nodes are still active
in_drain = [node for node in draining
if not node.is_done_draining(now, timeout)]
removes = [remove_node_from_lb(node=node)
for node in (set(nodes) - set(to_drain) - set(in_drain))]
changes = [drain_lb_node(node=node) for node in to_drain]
retry = (
[ConvergeLater(reasons=[ErrorReason.String('draining servers')])]
if in_drain else [])
return removes + changes + retry
def as_effect(self):
"""
Produce a :obj:`Effect` to remove some nodes from some RCv3 load
balancers.
"""
eff = rcv3.bulk_delete(self.lb_node_pairs)
return eff.on(success=lambda _: (StepResult.RETRY, [
ErrorReason.String(
'must re-gather after RCv3 LB change in order to update the '
'active cache')
]),
error=_failure_reporter(rcv3.BulkErrors))
def test_uses_step_request(self):
"""Steps are converted to requests."""
steps = [
TestStep(Effect(Constant((StepResult.SUCCESS, 'foo')))),
TestStep(Effect(Error(RuntimeError('uh oh'))))
]
effect = steps_to_effect(steps)
self.assertIs(type(effect.intent), ParallelEffects)
expected_exc_info = matches(MatchesException(RuntimeError('uh oh')))
self.assertEqual(
sync_perform(test_dispatcher(), effect),
[(StepResult.SUCCESS, 'foo'),
(StepResult.RETRY, [ErrorReason.Exception(expected_exc_info)])])
def test_change_load_balancer_node(self):
"""
:obj:`ChangeCLBNode.as_effect` produces a request for
modifying a load balancer node.
"""
eff = self._change_node_eff()
retry_result = (
StepResult.RETRY,
[ErrorReason.String(
'must re-gather after CLB change in order to update the '
'active cache')])
seq = [(eff.intent, lambda i: (StubResponse(202, {}), {}))]
self.assertEqual(perform_sequence(seq, eff), retry_result)
def test_failures_and_retries(self):
"""
If `rcv3.bulk_add` results in BulkErrors with
ServerUnprocessableError and other errors in it then step returns
FAILURE
"""
excp1 = rcv3.LBInactive("l1")
excp2 = rcv3.ServerUnprocessableError("s2")
seq = [(("ba", self.pairs), self.ba_raiser(excp1, excp2))]
self.assertEqual(
perform_sequence(seq, self.step.as_effect()),
(StepResult.FAILURE, [ErrorReason.String(excp1.message)])
)
def test_ensure_retry(self):
"""Tests that retry will be returned."""
seq = [
(delete_stack(stack_id='foo', stack_name='bar').intent,
lambda _: (StubResponse(204, ''), None)),
(Log('request-delete-stack', ANY), lambda _: None)
]
foo_stack = stack(id='foo', name='bar')
delete = DeleteStack(foo_stack)
reason = ('Waiting for stack to delete')
result = perform_sequence(seq, delete.as_effect())
self.assertEqual(result,
(StepResult.RETRY, [ErrorReason.String(reason)]))
def test_other_errors(self):
"""
Any error other than `BulkErrors` results in RETRY
"""
non_terminals = (ValueError("internal"),
APIError(code=500, body="why?"),
APIError(code=503, body="bad service"))
eff = self.step.as_effect()
for exc in non_terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.RETRY, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_failure(self):
"""
Returns FAILURE if rcv3.bulk_delete raises BulkErrors
"""
terminals = (rcv3.BulkErrors([rcv3.LBInactive("l1")]),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self.step.as_effect()
for exc in terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.FAILURE, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_add_nodes_to_clb_success_response_codes(self):
"""
:obj:`AddNodesToCLB` succeeds on 202.
"""
eff = self._add_one_node_to_clb()
seq = SequenceDispatcher([
(eff.intent, lambda i: (StubResponse(202, {}), '')),
(Log(ANY, ANY), lambda _: None)
])
expected = (
StepResult.RETRY,
[ErrorReason.String('must re-gather after adding to CLB in order '
'to update the active cache')])
with seq.consume():
self.assertEquals(sync_perform(seq, eff), expected)
def test_remove_nodes_from_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if there are any 4xx errors, then
the error is propagated up and the result is a failure.
"""
terminals = (APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def _assert_create_server_with_errs_has_status(self, exceptions, status):
"""
Helper function to make a :class:`CreateServer` effect, and resolve
it with the provided exceptions, asserting that the result is the
provided status, with the reason being the exception.
"""
eff = CreateServer(
server_config=freeze({'server': {'flavorRef': '1'}})).as_effect()
eff = resolve_effect(eff, 'random-name')
for exc in exceptions:
self.assertEqual(
resolve_effect(eff, service_request_error_response(exc),
is_error=True),
(status, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))])
)
def test_normal_use(self):
"""Tests normal usage."""
stack_config = pmap({'stack_name': 'baz', 'foo': 'bar'})
new_stack_config = pmap({'stack_name': 'baz_foo', 'foo': 'bar'})
self.create = CreateStack(stack_config)
self.seq = [
(Func(uuid4), lambda _: 'foo'),
(create_stack(thaw(new_stack_config)).intent,
lambda _: (StubResponse(200, {}), {'stack': {}})),
(Log('request-create-stack', ANY), lambda _: None)
]
reason = 'Waiting for stack to create'
result = perform_sequence(self.seq, self.create.as_effect())
self.assertEqual(result,
(StepResult.RETRY, [ErrorReason.String(reason)]))
def test_delete_server(self, mock_dav):
"""
:obj:`DeleteServer.as_effect` calls `delete_and_verify` with
retries. It returns SUCCESS on completion and RETRY on failure
"""
mock_dav.side_effect = lambda sid: Effect(sid)
eff = DeleteServer(server_id='abc123').as_effect()
self.assertIsInstance(eff.intent, Retry)
self.assertEqual(
eff.intent.should_retry,
ShouldDelayAndRetry(can_retry=retry_times(3),
next_interval=exponential_backoff_interval(2)))
self.assertEqual(eff.intent.effect.intent, 'abc123')
self.assertEqual(
resolve_effect(eff, (None, {})),
(StepResult.RETRY,
[ErrorReason.String('must re-gather after deletion in order to '
'update the active cache')]))
def test_remove_nodes_from_clb_non_terminal_failures_to_retry(self):
"""
:obj:`RemoveNodesFromCLB` retries if the CLB is temporarily locked,
or if the request was rate-limited, or if there was an API error and
the error is unknown but not a 4xx.
"""
non_terminals = (CLBImmutableError(lb_id=u"12345"),
CLBRateLimitError(lb_id=u"12345"),
APIError(code=500, body="oops!"),
TypeError("You did something wrong in your code."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in non_terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.RETRY, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_add_nodes_to_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or
if there is any other 4xx error, then
the error is propagated up and the result is a failure.
"""
terminals = (CLBNotFoundError(lb_id=u"12345"),
CLBDeletedError(lb_id=u"12345"),
NoSuchCLBError(lb_id=u"12345"),
CLBNodeLimitError(lb_id=u"12345", node_limit=25),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self._add_one_node_to_clb()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))