def test_retry_sequence_fails_if_mismatch_sequence(self):
"""
Fail if the wrong number of performers are given.
"""
r = Retry(
effect=Effect(1),
should_retry=ShouldDelayAndRetry(
can_retry=retry_times(5),
next_interval=repeating_interval(10)))
seq = [
retry_sequence(r, [lambda _: raise_(Exception()),
lambda _: raise_(Exception())])
]
self.assertRaises(AssertionError,
perform_sequence, seq, Effect(r))
def test_set_metadata_item(self):
"""
:obj:`SetMetadataItemOnServer.as_effect` produces a request for
setting a metadata item on a particular server. It succeeds if
successful, but does not fail for any errors.
"""
server_id = u'abc123'
meta = SetMetadataItemOnServer(server_id=server_id, key='metadata_key',
value='teapot')
eff = meta.as_effect()
seq = [
(eff.intent, lambda i: (StubResponse(202, {}), {})),
(Log(ANY, ANY), lambda _: None)
]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.SUCCESS, []))
exceptions = (NoSuchServerError("msg", server_id=server_id),
ServerMetadataOverLimitError("msg", server_id=server_id),
NovaRateLimitError("msg"),
APIError(code=500, body="", headers={}))
for exception in exceptions:
self.assertRaises(
type(exception),
perform_sequence,
[(eff.intent, lambda i: raise_(exception))],
eff)
def test_retry_sequence_retries_without_delays(self):
"""
Perform the wrapped effect with the performers given,
without any delay even if the original intent had a delay.
"""
r = Retry(
effect=Effect(1),
should_retry=ShouldDelayAndRetry(
can_retry=retry_times(5),
next_interval=repeating_interval(10)))
seq = [
retry_sequence(r, [lambda _: raise_(Exception()),
lambda _: raise_(Exception()),
lambda _: "yay done"])
]
self.assertEqual(perform_sequence(seq, Effect(r)), "yay done")
def test_buckets_acquired_errors(self):
"""
Errors raised from performing the converge_all_groups effect are
logged, and None is the ultimate result.
"""
def converge_all_groups(currently_converging, recent, _my_buckets,
all_buckets, divergent_flags, build_timeout,
interval):
return Effect('converge-all')
bound_sequence = [
(GetChildren(CONVERGENCE_DIRTY_DIR),
lambda i: ['flag1', 'flag2']),
('converge-all', lambda i: raise_(RuntimeError('foo'))),
(LogErr(
CheckFailureValue(RuntimeError('foo')),
'converge-all-groups-error', {}), noop)
]
sequence = self._log_sequence(bound_sequence)
# relying on the side-effect of setting up self.fake_partitioner
self._converger(converge_all_groups, dispatcher=sequence)
with sequence.consume():
result, = self.fake_partitioner.got_buckets([0])
self.assertEqual(self.successResultOf(result), None)
def test_ignores_errors(self):
"""
Errors are not logged and are propogated
"""
seq = [("internal", lambda i: raise_(ValueError("oops")))]
self.assertRaises(ValueError, perform_sequence, seq,
msg_with_time("mt", Effect("internal")), self.disp)
self.assertFalse(self.log.msg.called)
def test_ignores_errors(self):
"""
Errors are not logged and are propogated
"""
seq = [("internal", lambda i: raise_(ValueError("oops")))]
self.assertRaises(
ValueError, perform_sequence, seq,
msg_with_time("mt", Effect("internal")), self.disp)
self.assertFalse(self.log.msg.called)
def test_change_clb_node_terminal_errors(self):
"""Some errors during :obj:`ChangeCLBNode` make convergence fail."""
eff = self._change_node_eff()
terminal = (NoSuchCLBNodeError(lb_id=u'abc123', node_id=u'node1'),
CLBNotFoundError(lb_id=u'abc123'),
APIError(code=400, body="", headers={}))
for exception in terminal:
self.assertEqual(
perform_sequence([(eff.intent, lambda i: raise_(exception))],
eff),
(StepResult.FAILURE, [ANY]))
def test_change_clb_node_nonterminal_errors(self):
"""Some errors during :obj:`ChangeCLBNode` make convergence retry."""
eff = self._change_node_eff()
nonterminal = (APIError(code=500, body="", headers={}),
CLBNotActiveError(lb_id=u'abc123'),
CLBRateLimitError(lb_id=u'abc123'))
for exception in nonterminal:
self.assertEqual(
perform_sequence([(eff.intent, lambda i: raise_(exception))],
eff),
(StepResult.RETRY, ANY))
def test_failure(self):
"""
If setting divergent flag errors, then error is logged and raised
"""
seq = [
(CreateOrSet(path="/groups/divergent/t_g", content="dirty"),
lambda i: raise_(ValueError("oops"))),
(LogErr(CheckFailureValue(ValueError("oops")),
"mark-dirty-failure", {}),
noop)
]
self.assertRaises(
ValueError, perform_sequence, seq, trigger_convergence("t", "g"))
def test_can_have_a_different_should_retry_function(self):
"""
The ``should_retry`` function does not have to be a
:obj:`ShouldDelayAndRetry`.
"""
expected = Retry(effect=Effect(1), should_retry=ANY)
actual = Retry(effect=Effect(1), should_retry=lambda _: False)
seq = [
retry_sequence(expected, [lambda _: raise_(Exception())])
]
self.assertRaises(Exception,
perform_sequence, seq, Effect(actual))
def test_delete_node_other_error(self):
"""When marking clean raises arbitrary errors, an error is logged."""
sequence = [
(('ec', self.tenant_id, self.group_id, 3600),
lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)),
(DeleteNode(path='/groups/divergent/tenant-id_g1',
version=self.version),
lambda i: raise_(ZeroDivisionError())),
(LogErr(CheckFailureValue(ZeroDivisionError()),
'mark-clean-failure',
dict(path='/groups/divergent/tenant-id_g1',
dirty_version=self.version)), noop)
]
self._verify_sequence(sequence)
def test_error_validating_observer(self):
"""
The observer returned replaces event with error if it fails to
type check
"""
wrapper = SpecificationObserverWrapper(
self.observer, lambda e: raise_(ValueError('hm')))
wrapper({'message': ("something-bad",), 'a': 'b'})
self.assertEqual(
self.e,
[{'original_event': {'message': ("something-bad",), 'a': 'b'},
'isError': True,
'failure': CheckFailureValue(ValueError('hm')),
'why': 'Error validating event',
'message': ()}])
def test_other_errors(self):
"""
Any error other than `BulkErrors` results in RETRY
"""
non_terminals = (ValueError("internal"),
APIError(code=500, body="why?"),
APIError(code=503, body="bad service"))
eff = self.step.as_effect()
for exc in non_terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.RETRY, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_failure(self):
"""
Returns FAILURE if rcv3.bulk_delete raises BulkErrors
"""
terminals = (rcv3.BulkErrors([rcv3.LBInactive("l1")]),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self.step.as_effect()
for exc in terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.FAILURE, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_delete_node_not_found(self):
"""
When DeleteNode raises a NoNodeError, a message is logged and nothing
else is cleaned up.
"""
sequence = [
(('ec', self.tenant_id, self.group_id, 3600),
lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)),
(DeleteNode(path='/groups/divergent/tenant-id_g1',
version=self.version),
lambda i: raise_(NoNodeError())),
(Log('mark-clean-not-found',
dict(path='/groups/divergent/tenant-id_g1',
dirty_version=self.version)), noop)
]
self._verify_sequence(sequence)
def test_remove_nodes_from_clb_success_failures(self):
"""
:obj:`AddNodesToCLB` succeeds if the CLB is not in existence (has been
deleted or is not found).
"""
successes = [CLBNotFoundError(lb_id=u'12345'),
CLBDeletedError(lb_id=u'12345'),
NoSuchCLBError(lb_id=u'12345')]
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in successes:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(sync_perform(seq, eff),
(StepResult.SUCCESS, []))
def test_do_not_have_to_expect_an_exact_can_retry(self):
"""
The expected retry intent does not actually have to specify the
exact ``can_retry`` function, since it might just be a lambda,
which is hard to compare or hash.
"""
expected = Retry(effect=Effect(1), should_retry=ANY)
actual = Retry(effect=Effect(1), should_retry=ShouldDelayAndRetry(
can_retry=lambda _: False,
next_interval=repeating_interval(10)))
seq = [
retry_sequence(expected, [lambda _: raise_(Exception())])
]
self.assertRaises(Exception,
perform_sequence, seq, Effect(actual))
def test_remove_nodes_from_clb_retry(self):
"""
:obj:`RemoveNodesFromCLB`, on receiving a 400, parses out the nodes
that are no longer on the load balancer, and retries the bulk delete
with those nodes removed.
TODO: this has been left in as a regression test - this can probably be
removed the next time it's touched, as this functionality happens
in cloud_client now and there is a similar test there.
"""
lb_id = "12345"
node_ids = [str(i) for i in range(5)]
error_body = {
"validationErrors": {
"messages": [
"Node ids 1,2,3 are not a part of your loadbalancer"
]
},
"message": "Validation Failure",
"code": 400,
"details": "The object is not valid"
}
expected_req = service_request(
ServiceType.CLOUD_LOAD_BALANCERS,
'DELETE',
'loadbalancers/12345/nodes',
params={'id': transform_eq(sorted, node_ids)},
success_pred=ANY,
json_response=True).intent
expected_req2 = service_request(
ServiceType.CLOUD_LOAD_BALANCERS,
'DELETE',
'loadbalancers/12345/nodes',
params={'id': transform_eq(sorted, ['0', '4'])},
success_pred=ANY,
json_response=True).intent
step = RemoveNodesFromCLB(lb_id=lb_id, node_ids=pset(node_ids))
seq = [
(expected_req,
lambda i: raise_(APIError(400, json.dumps(error_body)))),
(expected_req2, lambda i: stub_pure_response('', 202)),
]
r = perform_sequence(seq, step.as_effect())
self.assertEqual(r, (StepResult.SUCCESS, []))
def test_delete_node_version_mismatch(self):
"""
When the version of the dirty flag changes during a call to
converge_one_group, and DeleteNode raises a BadVersionError, the error
is logged and nothing else is cleaned up.
"""
sequence = [
(('ec', self.tenant_id, self.group_id, 3600),
lambda i: (StepResult.SUCCESS, ScalingGroupStatus.ACTIVE)),
(DeleteNode(path='/groups/divergent/tenant-id_g1',
version=self.version),
lambda i: raise_(BadVersionError())),
(Log('mark-clean-skipped',
dict(path='/groups/divergent/tenant-id_g1',
dirty_version=self.version)), noop)
]
self._verify_sequence(sequence)
def test_no_scaling_group(self):
"""
When the scaling group disappears, a fatal error is logged and the
dirty flag is cleaned up.
"""
expected_error = NoSuchScalingGroupError(self.tenant_id, self.group_id)
sequence = [
(('ec', self.tenant_id, self.group_id, 3600),
lambda i: raise_(expected_error)),
(LogErr(CheckFailureValue(expected_error),
'converge-fatal-error', {}),
noop),
(DeleteNode(path='/groups/divergent/tenant-id_g1',
version=self.version), noop),
(Log('mark-clean-success', {}), noop)
]
self._verify_sequence(sequence)
def test_error_writing(self):
"""
Logs and ignores error writing to the file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), const(["2", "0.0"])),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [7, 86400.0]),
lambda i: raise_(IOError("bad"))),
(LogErr(mock.ANY, "error updating number of tenants", {}), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 9)],
self.groups))
def test_remove_nodes_from_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if there are any 4xx errors, then
the error is propagated up and the result is a failure.
"""
terminals = (APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_no_last_info(self):
"""
Returns first 5 non-convergence tenants if could not fetch last info
from file
"""
seq = [
(GetAllGroups(), const(self.groups)),
(ReadFileLines("file"), lambda i: raise_(IOError("e"))),
(LogErr(mock.ANY, "error reading previous number of tenants", {}),
noop),
(Func(datetime.utcnow), const(datetime(1970, 1, 2))),
(WriteFileLines("file", [5, 86400.0]), noop)
]
r = perform_sequence(seq, get_todays_scaling_groups(["t1"], "file"))
self.assertEqual(
r,
keyfilter(lambda k: k in ["t{}".format(i) for i in range(1, 7)],
self.groups))
def test_error_validating_observer(self):
"""
The observer returned replaces event with error if it fails to
type check
"""
wrapper = SpecificationObserverWrapper(self.observer, lambda e: raise_(ValueError("hm")))
wrapper({"message": ("something-bad",), "a": "b"})
self.assertEqual(
self.e,
[
{
"original_event": {"message": ("something-bad",), "a": "b"},
"isError": True,
"failure": CheckFailureValue(ValueError("hm")),
"why": "Error validating event",
"message": (),
}
],
)
def test_error_validating_observer(self):
"""
The observer returned replaces event with error if it fails to
type check
"""
wrapper = SpecificationObserverWrapper(
self.observer, lambda e: raise_(ValueError('hm')))
wrapper({'message': ("something-bad", ), 'a': 'b'})
self.assertEqual(self.e,
[{
'original_event': {
'message': ("something-bad", ),
'a': 'b'
},
'isError': True,
'failure': CheckFailureValue(ValueError('hm')),
'why': 'Error validating event',
'message': ()
}])
def test_remove_nodes_from_clb_non_terminal_failures_to_retry(self):
"""
:obj:`RemoveNodesFromCLB` retries if the CLB is temporarily locked,
or if the request was rate-limited, or if there was an API error and
the error is unknown but not a 4xx.
"""
non_terminals = (CLBImmutableError(lb_id=u"12345"),
CLBRateLimitError(lb_id=u"12345"),
APIError(code=500, body="oops!"),
TypeError("You did something wrong in your code."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in non_terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.RETRY, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_add_nodes_to_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or
if there is any other 4xx error, then
the error is propagated up and the result is a failure.
"""
terminals = (CLBNotFoundError(lb_id=u"12345"),
CLBDeletedError(lb_id=u"12345"),
NoSuchCLBError(lb_id=u"12345"),
CLBNodeLimitError(lb_id=u"12345", node_limit=25),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self._add_one_node_to_clb()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_add_event_only_retries_5_times_on_non_4xx_api_errors(self):
"""
Attempting to add an event is only retried up to a maximum of 5 times,
and only if it's not an 4XX APIError.
"""
responses = [
lambda _: raise_(Exception("oh noes!")),
lambda _: raise_(ResponseFailed(Failure(Exception(":(")))),
lambda _: raise_(APIError(code=100, body="<some xml>")),
lambda _: raise_(APIError(code=202, body="<some xml>")),
lambda _: raise_(APIError(code=301, body="<some xml>")),
lambda _: raise_(APIError(code=501, body="<some xml>")),
]
with self.assertRaises(APIError) as cm:
self._perform_add_event(responses)
self.assertEqual(cm.exception.code, 501)
def test_first_error_extraction(self):
"""
If the GetScalingGroupInfo effect fails, its exception is raised
directly, without the FirstError wrapper.
"""
# Perform the GetScalingGroupInfo by raising an exception
sequence = [
(Log("begin-convergence", {}), noop),
(Func(datetime.utcnow), lambda i: self.now),
(MsgWithTime("gather-convergence-data", mock.ANY),
nested_sequence([
parallel_sequence([
[(self.gsgi, lambda i: raise_(RuntimeError('foo')))],
[("anything", noop)]
])
]))
]
# And make sure that exception isn't wrapped in FirstError.
e = self.assertRaises(
RuntimeError, perform_sequence, sequence, self._invoke(),
test_dispatcher())
self.assertEqual(str(e), 'foo')
def test_unexpected_errors(self):
"""
Unexpected exceptions log a non-fatal error and don't clean up the
dirty flag.
"""
converging = Reference(pset())
recent = Reference(pmap())
expected_error = RuntimeError('oh no!')
sequence = [
(ReadReference(converging), lambda i: pset()),
add_to_currently(converging, self.group_id),
(('ec', self.tenant_id, self.group_id, 3600),
lambda i: raise_(expected_error)),
(Func(time.time), lambda i: 100),
add_to_recently(recent, self.group_id, 100),
(ModifyReference(converging,
match_func(pset([self.group_id]), pset())),
noop),
(LogErr(CheckFailureValue(expected_error),
'converge-non-fatal-error', {}),
noop),
]
self._verify_sequence(sequence, converging=converging, recent=recent,
allow_refs=False)
def ba_raiser(self, *errors):
return lambda i: raise_(rcv3.BulkErrors(errors))
def test_add_event_bails_on_4xx_api_errors(self):
"""
If CF returns a 4xx error, adding an event is not retried.
"""
response = [lambda _: raise_(APIError(code=409, body="<some xml>"))]
self.assertRaises(APIError, self._perform_add_event, response)
