def test_retry_sequence_fails_if_mismatch_sequence(self):
"""
Fail if the wrong number of performers are given.
"""
r = Retry(
effect=Effect(1),
should_retry=ShouldDelayAndRetry(
can_retry=retry_times(5),
next_interval=repeating_interval(10)))
seq = [
retry_sequence(r, [lambda _: raise_(Exception()),
lambda _: raise_(Exception())])
]
self.assertRaises(AssertionError,
perform_sequence, seq, Effect(r))
def test_set_metadata_item(self):
"""
:obj:`SetMetadataItemOnServer.as_effect` produces a request for
setting a metadata item on a particular server. It succeeds if
successful, but does not fail for any errors.
"""
server_id = u'abc123'
meta = SetMetadataItemOnServer(server_id=server_id, key='metadata_key',
value='teapot')
eff = meta.as_effect()
seq = [
(eff.intent, lambda i: (StubResponse(202, {}), {})),
(Log(ANY, ANY), lambda _: None)
]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.SUCCESS, []))
exceptions = (NoSuchServerError("msg", server_id=server_id),
ServerMetadataOverLimitError("msg", server_id=server_id),
NovaRateLimitError("msg"),
APIError(code=500, body="", headers={}))
for exception in exceptions:
self.assertRaises(
type(exception),
perform_sequence,
[(eff.intent, lambda i: raise_(exception))],
eff)
def test_retry_sequence_retries_without_delays(self):
"""
Perform the wrapped effect with the performers given,
without any delay even if the original intent had a delay.
"""
r = Retry(
effect=Effect(1),
should_retry=ShouldDelayAndRetry(
can_retry=retry_times(5),
next_interval=repeating_interval(10)))
seq = [
retry_sequence(r, [lambda _: raise_(Exception()),
lambda _: raise_(Exception()),
lambda _: "yay done"])
]
self.assertEqual(perform_sequence(seq, Effect(r)), "yay done")
def test_ignores_errors(self):
"""
Errors are not logged and are propogated
"""
seq = [("internal", lambda i: raise_(ValueError("oops")))]
self.assertRaises(ValueError, perform_sequence, seq, msg_with_time("mt", Effect("internal")), self.disp)
self.assertFalse(self.log.msg.called)
def test_ignores_errors(self):
"""
Errors are not logged and are propogated
"""
seq = [("internal", lambda i: raise_(ValueError("oops")))]
self.assertRaises(ValueError, perform_sequence, seq,
msg_with_time("mt", Effect("internal")), self.disp)
self.assertFalse(self.log.msg.called)
def test_list_servers_details_all_propagates_errors(self):
"""
:func:`list_servers_details_all` propagates exceptions from making
the individual requests (from :func:`list_servers_details_page`).
"""
eff = list_servers_details_all({'marker': ['1']})
seq = [(self._list_server_details_intent({'marker': ['1']}),
lambda _: raise_(NovaComputeFaultError('error')))]
self.assertRaises(NovaComputeFaultError, perform_sequence, seq, eff)
def test_quit_game():
for exc in (KeyboardInterrupt(), EOFError()):
expected_effects = [
(Display(render(initial_state)), noop),
(Prompt("> "), lambda i: raise_(exc)),
(Display("\nThanks for playing!"), noop),
]
eff = step(initial_state)
with raises(SystemExit):
perform_sequence(expected_effects, eff)
def test_group_deleted(self):
"""
Does nothing if group has been deleted
"""
seq = [(GetScalingGroupInfo(tenant_id="tid", group_id="gid"),
lambda i: raise_(NoSuchScalingGroupError("tid", "gid"))),
(Log("selfheal-group-deleted",
dict(tenant_id="tid", scaling_group_id="gid")), noop)]
self.assertIsNone(
perform_sequence(seq, sh.check_and_trigger("tid", "gid")))
def test_list_servers_details_all_propagates_errors(self):
"""
:func:`list_servers_details_all` propagates exceptions from making
the individual requests (from :func:`list_servers_details_page`).
"""
eff = list_servers_details_all({'marker': ['1']})
seq = [
(self._list_server_details_intent({'marker': ['1']}),
lambda _: raise_(NovaComputeFaultError('error')))
]
self.assertRaises(NovaComputeFaultError, perform_sequence, seq, eff)
def test_change_clb_node_terminal_errors(self):
"""Some errors during :obj:`ChangeCLBNode` make convergence fail."""
eff = self._change_node_eff()
terminal = (NoSuchCLBNodeError(lb_id=u'abc123', node_id=u'node1'),
CLBNotFoundError(lb_id=u'abc123'),
APIError(code=400, body="", headers={}))
for exception in terminal:
self.assertEqual(
perform_sequence([(eff.intent, lambda i: raise_(exception))],
eff),
(StepResult.FAILURE, [ANY]))
def test_change_clb_node_nonterminal_errors(self):
"""Some errors during :obj:`ChangeCLBNode` make convergence retry."""
eff = self._change_node_eff()
nonterminal = (APIError(code=500, body="", headers={}),
CLBNotActiveError(lb_id=u'abc123'),
CLBRateLimitError(lb_id=u'abc123'))
for exception in nonterminal:
self.assertEqual(
perform_sequence([(eff.intent, lambda i: raise_(exception))],
eff),
(StepResult.RETRY, ANY))
def test_mainloop():
expected_effects = [(Display(render(initial_state)), noop),
(Prompt("> "), lambda i: "move east"),
(Display("Okay."), noop),
(SaveGame(state=in_street), noop),
(Display(render(in_street)), noop),
(Prompt("> "), lambda i: raise_(KeyboardInterrupt())),
(Display("\nThanks for playing!"), noop)]
eff = mainloop(initial_state)
with raises(SystemExit):
perform_sequence(expected_effects, eff)
def test_can_have_a_different_should_retry_function(self):
"""
The ``should_retry`` function does not have to be a
:obj:`ShouldDelayAndRetry`.
"""
expected = Retry(effect=Effect(1), should_retry=ANY)
actual = Retry(effect=Effect(1), should_retry=lambda _: False)
seq = [
retry_sequence(expected, [lambda _: raise_(Exception())])
]
self.assertRaises(Exception,
perform_sequence, seq, Effect(actual))
def test_group_deleted(self):
"""
Does nothing if group has been deleted
"""
seq = [
(GetScalingGroupInfo(tenant_id="tid", group_id="gid"),
lambda i: raise_(NoSuchScalingGroupError("tid", "gid"))),
(Log("selfheal-group-deleted",
dict(tenant_id="tid", scaling_group_id="gid")),
noop)
]
self.assertIsNone(
perform_sequence(seq, sh.check_and_trigger("tid", "gid")))
def remove_clb_nodes(lb_id, node_ids):
"""
Remove multiple nodes from a load balancer.
:param str lb_id: A load balancer ID.
:param node_ids: iterable of node IDs.
:return: Effect of None.
Succeeds on 202.
This function will handle the case where *some* of the nodes are valid and
some aren't, by retrying deleting only the valid ones.
"""
node_ids = list(node_ids)
partial = None
if len(node_ids) > CLB_BATCH_DELETE_LIMIT:
not_removing = node_ids[CLB_BATCH_DELETE_LIMIT:]
node_ids = node_ids[:CLB_BATCH_DELETE_LIMIT]
partial = CLBPartialNodesRemoved(six.text_type(lb_id),
map(six.text_type, not_removing),
map(six.text_type, node_ids))
eff = service_request(
ServiceType.CLOUD_LOAD_BALANCERS,
'DELETE',
append_segments('loadbalancers', lb_id, 'nodes'),
params={'id': map(str, node_ids)},
success_pred=has_code(202))
def check_invalid_nodes(exc_info):
code = exc_info[1].code
body = exc_info[1].body
if code == 400:
message = try_json_with_keys(
body, ["validationErrors", "messages", 0])
if message is not None:
match = _CLB_NODE_REMOVED_PATTERN.match(message)
if match:
removed = concat([group.split(',')
for group in match.groups()])
return remove_clb_nodes(lb_id,
set(node_ids) - set(removed))
six.reraise(*exc_info)
return eff.on(
error=catch(APIError, check_invalid_nodes)
).on(
error=_only_json_api_errors(
lambda c, b: _process_clb_api_error(c, b, lb_id))
).on(success=lambda _: None if partial is None else raise_(partial))
def test_error_validating_observer(self):
"""
The observer returned replaces event with error if it fails to
type check
"""
wrapper = SpecificationObserverWrapper(
self.observer, lambda e: raise_(ValueError('hm')))
wrapper({'message': ("something-bad",), 'a': 'b'})
self.assertEqual(
self.e,
[{'original_event': {'message': ("something-bad",), 'a': 'b'},
'isError': True,
'failure': CheckFailureValue(ValueError('hm')),
'why': 'Error validating event',
'message': ()}])
def test_other_errors(self):
"""
Any error other than `BulkErrors` results in RETRY
"""
non_terminals = (ValueError("internal"),
APIError(code=500, body="why?"),
APIError(code=503, body="bad service"))
eff = self.step.as_effect()
for exc in non_terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.RETRY, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_failure(self):
"""
Returns FAILURE if rcv3.bulk_delete raises BulkErrors
"""
terminals = (rcv3.BulkErrors([rcv3.LBInactive("l1")]),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self.step.as_effect()
for exc in terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.FAILURE, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_do_not_have_to_expect_an_exact_can_retry(self):
"""
The expected retry intent does not actually have to specify the
exact ``can_retry`` function, since it might just be a lambda,
which is hard to compare or hash.
"""
expected = Retry(effect=Effect(1), should_retry=ANY)
actual = Retry(effect=Effect(1), should_retry=ShouldDelayAndRetry(
can_retry=lambda _: False,
next_interval=repeating_interval(10)))
seq = [
retry_sequence(expected, [lambda _: raise_(Exception())])
]
self.assertRaises(Exception,
perform_sequence, seq, Effect(actual))
def test_remove_nodes_from_clb_success_failures(self):
"""
:obj:`AddNodesToCLB` succeeds if the CLB is not in existence (has been
deleted or is not found).
"""
successes = [CLBNotFoundError(lb_id=u'12345'),
CLBDeletedError(lb_id=u'12345'),
NoSuchCLBError(lb_id=u'12345')]
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in successes:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(sync_perform(seq, eff),
(StepResult.SUCCESS, []))
def test_remove_nodes_from_clb_retry(self):
"""
:obj:`RemoveNodesFromCLB`, on receiving a 400, parses out the nodes
that are no longer on the load balancer, and retries the bulk delete
with those nodes removed.
TODO: this has been left in as a regression test - this can probably be
removed the next time it's touched, as this functionality happens
in cloud_client now and there is a similar test there.
"""
lb_id = "12345"
node_ids = [str(i) for i in range(5)]
error_body = {
"validationErrors": {
"messages": [
"Node ids 1,2,3 are not a part of your loadbalancer"
]
},
"message": "Validation Failure",
"code": 400,
"details": "The object is not valid"
}
expected_req = service_request(
ServiceType.CLOUD_LOAD_BALANCERS,
'DELETE',
'loadbalancers/12345/nodes',
params={'id': transform_eq(sorted, node_ids)},
success_pred=ANY,
json_response=True).intent
expected_req2 = service_request(
ServiceType.CLOUD_LOAD_BALANCERS,
'DELETE',
'loadbalancers/12345/nodes',
params={'id': transform_eq(sorted, ['0', '4'])},
success_pred=ANY,
json_response=True).intent
step = RemoveNodesFromCLB(lb_id=lb_id, node_ids=pset(node_ids))
seq = [
(expected_req,
lambda i: raise_(APIError(400, json.dumps(error_body)))),
(expected_req2, lambda i: stub_pure_response('', 202)),
]
r = perform_sequence(seq, step.as_effect())
self.assertEqual(r, (StepResult.SUCCESS, []))
def test_remove_nodes_from_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if there are any 4xx errors, then
the error is propagated up and the result is a failure.
"""
terminals = (APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def remove_clb_nodes(lb_id, node_ids):
"""
Remove multiple nodes from a load balancer.
:param str lb_id: A load balancer ID.
:param node_ids: iterable of node IDs.
:return: Effect of None.
Succeeds on 202.
This function will handle the case where *some* of the nodes are valid and
some aren't, by retrying deleting only the valid ones.
"""
node_ids = list(node_ids)
partial = None
if len(node_ids) > CLB_BATCH_DELETE_LIMIT:
not_removing = node_ids[CLB_BATCH_DELETE_LIMIT:]
node_ids = node_ids[:CLB_BATCH_DELETE_LIMIT]
partial = CLBPartialNodesRemoved(six.text_type(lb_id),
map(six.text_type, not_removing),
map(six.text_type, node_ids))
eff = service_request(ServiceType.CLOUD_LOAD_BALANCERS,
'DELETE',
append_segments('loadbalancers', lb_id, 'nodes'),
params={'id': map(str, node_ids)},
success_pred=has_code(202))
def check_invalid_nodes(exc_info):
code = exc_info[1].code
body = exc_info[1].body
if code == 400:
message = try_json_with_keys(body,
["validationErrors", "messages", 0])
if message is not None:
match = _CLB_NODE_REMOVED_PATTERN.match(message)
if match:
removed = concat(
[group.split(',') for group in match.groups()])
return remove_clb_nodes(lb_id,
set(node_ids) - set(removed))
six.reraise(*exc_info)
return eff.on(error=catch(APIError, check_invalid_nodes)).on(
error=only_json_api_errors(
lambda c, b: _process_clb_api_error(c, b, lb_id))).on(
success=lambda _: None if partial is None else raise_(partial))
def test_error_validating_observer(self):
"""
The observer returned replaces event with error if it fails to
type check
"""
wrapper = SpecificationObserverWrapper(
self.observer, lambda e: raise_(ValueError('hm')))
wrapper({'message': ("something-bad", ), 'a': 'b'})
self.assertEqual(self.e,
[{
'original_event': {
'message': ("something-bad", ),
'a': 'b'
},
'isError': True,
'failure': CheckFailureValue(ValueError('hm')),
'why': 'Error validating event',
'message': ()
}])
def test_remove_nodes_from_clb_non_terminal_failures_to_retry(self):
"""
:obj:`RemoveNodesFromCLB` retries if the CLB is temporarily locked,
or if the request was rate-limited, or if there was an API error and
the error is unknown but not a 4xx.
"""
non_terminals = (CLBImmutableError(lb_id=u"12345"),
CLBRateLimitError(lb_id=u"12345"),
APIError(code=500, body="oops!"),
TypeError("You did something wrong in your code."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in non_terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.RETRY, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_add_nodes_to_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or
if there is any other 4xx error, then
the error is propagated up and the result is a failure.
"""
terminals = (CLBNotFoundError(lb_id=u"12345"),
CLBDeletedError(lb_id=u"12345"),
NoSuchCLBError(lb_id=u"12345"),
CLBNodeLimitError(lb_id=u"12345", node_limit=25),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self._add_one_node_to_clb()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_add_event_only_retries_5_times_on_non_4xx_api_errors(self):
"""
Attempting to add an event is only retried up to a maximum of 5 times,
and only if it's not an 4XX APIError.
"""
responses = [
lambda _: raise_(Exception("oh noes!")),
lambda _: raise_(ResponseFailed(Failure(Exception(":(")))),
lambda _: raise_(APIError(code=100, body="<some xml>")),
lambda _: raise_(APIError(code=202, body="<some xml>")),
lambda _: raise_(APIError(code=301, body="<some xml>")),
lambda _: raise_(APIError(code=501, body="<some xml>")),
]
with self.assertRaises(APIError) as cm:
self._perform_add_event(responses)
self.assertEqual(cm.exception.code, 501)
def test_add_event_only_retries_5_times_on_non_4xx_api_errors(self):
"""
Attempting to add an event is only retried up to a maximum of 5 times,
and only if it's not an 4XX APIError.
"""
responses = [
lambda _: raise_(Exception("oh noes!")),
lambda _: raise_(ResponseFailed(Failure(Exception(":(")))),
lambda _: raise_(APIError(code=100, body="<some xml>")),
lambda _: raise_(APIError(code=202, body="<some xml>")),
lambda _: raise_(APIError(code=301, body="<some xml>")),
lambda _: raise_(APIError(code=501, body="<some xml>")),
]
with self.assertRaises(APIError) as cm:
self._perform_add_event(responses)
self.assertEqual(cm.exception.code, 501)
def test_add_event_bails_on_4xx_api_errors(self):
"""
If CF returns a 4xx error, adding an event is not retried.
"""
response = [lambda _: raise_(APIError(code=409, body="<some xml>"))]
self.assertRaises(APIError, self._perform_add_event, response)
def ba_raiser(self, *errors):
return lambda i: raise_(rcv3.BulkErrors(errors))
def test_add_event_bails_on_4xx_api_errors(self):
"""
If CF returns a 4xx error, adding an event is not retried.
"""
response = [lambda _: raise_(APIError(code=409, body="<some xml>"))]
self.assertRaises(APIError, self._perform_add_event, response)
def conste(e):
"""
Like ``const`` but takes and exception and returns function that raises
the exception
"""
return lambda i: raise_(e)
