def test_authenticate_tenant_propagates_auth_errors(self):
"""
authenticate_tenant propagates errors from authenticate_user.
"""
self.impersonate_user.side_effect = lambda *a, **k: fail(
UpstreamError(Failure(APIError(401, '4')), 'identity', 'o'))
self.authenticate_user.side_effect = lambda *a, **kw: fail(
UpstreamError(Failure(APIError(500, '500')), 'identity', 'o'))
f = self.failureResultOf(self.ia.authenticate_tenant(111111),
UpstreamError)
self.assertEqual(f.value.reason.value.code, 500)
def test_failure(self):
"""
Returns FAILURE if rcv3.bulk_delete raises BulkErrors
"""
terminals = (rcv3.BulkErrors([rcv3.LBInactive("l1")]),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self.step.as_effect()
for exc in terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.FAILURE, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_other_errors(self):
"""
Any error other than `BulkErrors` results in RETRY
"""
non_terminals = (ValueError("internal"),
APIError(code=500, body="why?"),
APIError(code=503, body="bad service"))
eff = self.step.as_effect()
for exc in non_terminals:
seq = [(("bd", self.pairs), lambda i: raise_(exc))]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.RETRY, [
ErrorReason.Exception((type(exc), exc, ANY))])
)
def test_diagnose_unwraps_first_error_if_apierr_or_connection_error(self):
"""
:func:`diagnose` unwraps :class:`FirstError`, no matter how deeply
nested, and wraps the underlying :class:`ConnectionRefusedError` and
:class:`APIError` in an :class:`UpstreamError`
"""
def _wrap(exception):
return lambda: fail(
FirstError(
Failure(FirstError(Failure(exception), 0)),
0))
f = self.failureResultOf(
diagnose("system", "operation")(_wrap(APIError(200, {})))(),
UpstreamError)
self.assertTrue(f.value.reason.check(APIError))
self.assertEqual(f.value.system, "system")
self.assertEqual(f.value.operation, "operation")
f = self.failureResultOf(
diagnose("system", "operation")(
_wrap(ConnectionRefusedError('meh')))(),
UpstreamError)
self.assertTrue(f.value.reason.check(ConnectionRefusedError))
self.assertEqual(f.value.system, "system")
self.assertEqual(f.value.operation, "operation")
def test_authenticate_tenant_retries_getting_endpoints_for_the_impersonation_token(
self):
"""
authenticate_tenant fetches all the endpoints for the impersonation and
retries with new authentication token if it gets 401
"""
self.endpoints_for_token.side_effect = [
fail(UpstreamError(Failure(APIError(401, '')), 'identity', 'o')),
succeed({'endpoints': [{
'name': 'anEndpoint',
'type': 'anType'
}]})
]
self.successResultOf(self.ia.authenticate_tenant(111111, log=self.log))
self.endpoints_for_token.assert_has_calls([
mock.call(self.admin_url,
None,
'impersonation_token',
log=self.log),
mock.call(self.admin_url,
'auth-token',
'impersonation_token',
log=self.log)
])
self.authenticate_user.assert_called_once_with(self.url,
self.user,
self.password,
log=self.log)
self.log.msg.assert_called_once_with(
'Getting new identity admin token')
def test_set_metadata_item(self):
"""
:obj:`SetMetadataItemOnServer.as_effect` produces a request for
setting a metadata item on a particular server. It succeeds if
successful, but does not fail for any errors.
"""
server_id = u'abc123'
meta = SetMetadataItemOnServer(server_id=server_id, key='metadata_key',
value='teapot')
eff = meta.as_effect()
seq = [
(eff.intent, lambda i: (StubResponse(202, {}), {})),
(Log(ANY, ANY), lambda _: None)
]
self.assertEqual(
perform_sequence(seq, eff),
(StepResult.SUCCESS, []))
exceptions = (NoSuchServerError("msg", server_id=server_id),
ServerMetadataOverLimitError("msg", server_id=server_id),
NovaRateLimitError("msg"),
APIError(code=500, body="", headers={}))
for exception in exceptions:
self.assertRaises(
type(exception),
perform_sequence,
[(eff.intent, lambda i: raise_(exception))],
eff)
def test_remove_nodes_from_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if there are any 4xx errors, then
the error is propagated up and the result is a failure.
"""
terminals = (APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_authenticate_tenant_propagates_endpoint_list_errors(self):
"""
authenticate_tenant propagates errors from endpoints_for_token
"""
self.endpoints_for_token.return_value = fail(APIError(500, '500'))
failure = self.failureResultOf(self.ia.authenticate_tenant(111111))
self.assertTrue(failure.check(APIError))
def test_authenticate_tenant_propagates_impersonation_errors(self):
"""
authenticate_tenant propagates errors from impersonate_user
"""
self.impersonate_user.return_value = fail(APIError(500, '500'))
failure = self.failureResultOf(self.ia.authenticate_tenant(111111))
self.assertTrue(failure.check(APIError))
def test_add_event_only_retries_5_times_on_non_4xx_api_errors(self):
"""
Attempting to add an event is only retried up to a maximum of 5 times,
and only if it's not an 4XX APIError.
"""
responses = [
lambda _: raise_(Exception("oh noes!")),
lambda _: raise_(ResponseFailed(Failure(Exception(":(")))),
lambda _: raise_(APIError(code=100, body="<some xml>")),
lambda _: raise_(APIError(code=202, body="<some xml>")),
lambda _: raise_(APIError(code=301, body="<some xml>")),
lambda _: raise_(APIError(code=501, body="<some xml>")),
]
with self.assertRaises(APIError) as cm:
self._perform_add_event(responses)
self.assertEqual(cm.exception.code, 501)
def test_auth_failure_propagated_to_caller(self):
"""
authenticate_tenant propagates auth failures to the caller.
"""
self.resps[1] = APIError(500, '500')
d = self.ca.authenticate_tenant(1)
self.failureResultOf(d, APIError)
def test_auth_failure_propagated_to_caller(self):
"""
authenticate_tenant propagates auth failures to the caller.
"""
self.auth_function.side_effect = lambda _: fail(APIError(500, '500'))
d = self.ca.authenticate_tenant(1)
failure = self.failureResultOf(d)
self.assertTrue(failure.check(APIError))
def test_raise_error_on_code_does_not_match_code(self):
"""
``raise_error_on_code`` expects an APIError, and raises a particular
error given a specific code. Otherwise, it just wraps it in a
:class:`RequestError`
"""
failure = Failure(APIError(404, '', {}))
self.assertRaises(RequestError, raise_error_on_code,
failure, 500, DummyException(), 'url')
def test_authenticate_tenant_propagates_endpoint_list_errors(self):
"""
authenticate_tenant propagates errors from endpoints_for_token
"""
self.endpoints_for_token.side_effect = lambda *a, **kw: fail(
UpstreamError(Failure(APIError(500, '500')), 'identity', 'o'))
f = self.failureResultOf(self.ia.authenticate_tenant(111111),
UpstreamError)
self.assertEqual(f.value.reason.value.code, 500)
def test_authenticate_tenant_propagates_user_list_errors(self):
"""
authenticate_tenant propagates errors from user_for_tenant
"""
self.authenticate_user.side_effect = lambda *a, **kw: fail(
UpstreamError(Failure(APIError(500, '500')), 'identity', 'o'))
f = self.failureResultOf(self.st.authenticate_tenant('111111'),
UpstreamError)
self.assertEqual(f.value.reason.value.code, 500)
def test_500_error(self):
"""
Any error other than 401 is propogated
"""
d = retry_on_unauth(
lambda: fail(UpstreamError(Failure(APIError(500, 'a')), 'identity', 'o')),
self.auth)
f = self.failureResultOf(d, UpstreamError)
f.value.reason.trap(APIError)
self.assertEqual(f.value.reason.value.code, 500)
def test_auth_error_propogates(self):
"""
`auth()` errors are propogated
"""
auth = mock.Mock(spec=[], return_value=fail(ValueError('a')))
func = mock.Mock(side_effect=[
fail(UpstreamError(Failure(APIError(401, '401')), 'identity', 'o')),
succeed('r')])
d = retry_on_unauth(func, auth)
self.failureResultOf(d, ValueError)
auth.assert_called_once_with()
def test_apierror_identity(self):
"""
Wraps APIError from identity and parses error body accordingly
"""
body = json.dumps({"identityFault": {"message": "ba"}})
apie = APIError(410, body, {})
err = UpstreamError(Failure(apie), 'identity', 'stuff', 'xkcd.com')
self.assertEqual(str(err), 'identity error: 410 - ba (stuff)')
self.assertEqual(err.details, {
'system': 'identity', 'operation': 'stuff', 'url': 'xkcd.com',
'message': 'ba', 'code': 410, 'body': body, 'headers': {}})
def test_auth_error(self):
"""
Calls auth() followed by func() again if initial func() returns 401
"""
auth = mock.Mock(spec=[], return_value=succeed('a'))
func = mock.Mock(side_effect=[
fail(UpstreamError(Failure(APIError(401, '401')), 'identity', 'o')),
succeed('r')])
d = retry_on_unauth(func, auth)
self.assertEqual(self.successResultOf(d), 'r')
auth.assert_called_once_with()
def test_apierror_nova(self):
"""
Wraps APIError from nova and parses error body accordingly
"""
body = json.dumps({"computeFault": {"message": "b"}})
apie = APIError(404, body, {})
err = UpstreamError(Failure(apie), 'nova', 'add', 'xkcd.com')
self.assertEqual(str(err), 'nova error: 404 - b (add)')
self.assertEqual(err.details, {
'system': 'nova', 'operation': 'add', 'url': 'xkcd.com',
'message': 'b', 'code': 404, 'body': body, 'headers': {}})
def test_apierror_clb(self):
"""
Wraps APIError from clb and parses error body accordingly
"""
body = json.dumps({"message": "b"})
apie = APIError(403, body, {'h1': 2})
err = UpstreamError(Failure(apie), 'clb', 'remove', 'xkcd.com')
self.assertEqual(str(err), 'clb error: 403 - b (remove)')
self.assertEqual(err.details, {
'system': 'clb', 'operation': 'remove', 'url': 'xkcd.com',
'message': 'b', 'code': 403, 'body': body, 'headers': {'h1': 2}})
def test_api_error(self):
"""
An APIError will be instantiated with an HTTP Code and an HTTP response
body and will expose these in public attributes and have a reasonable
string representation.
"""
e = APIError(404, "Not Found.")
self.assertEqual(e.code, 404)
self.assertEqual(e.body, "Not Found.")
self.assertEqual(str(e), "API Error code=404, body='Not Found.'")
def test_delete_and_verify_del_fails(self):
"""
:func:`delete_and_verify` fails if delete server fails
"""
eff = delete_and_verify('sid')
self.assertRaises(
APIError,
resolve_effect,
eff,
service_request_error_response(APIError(500, '')),
is_error=True)
def test_change_clb_node_nonterminal_errors(self):
"""Some errors during :obj:`ChangeCLBNode` make convergence retry."""
eff = self._change_node_eff()
nonterminal = (APIError(code=500, body="", headers={}),
CLBNotActiveError(lb_id=u'abc123'),
CLBRateLimitError(lb_id=u'abc123'))
for exception in nonterminal:
self.assertEqual(
perform_sequence([(eff.intent, lambda i: raise_(exception))],
eff),
(StepResult.RETRY, ANY))
def test_change_clb_node_terminal_errors(self):
"""Some errors during :obj:`ChangeCLBNode` make convergence fail."""
eff = self._change_node_eff()
terminal = (NoSuchCLBNodeError(lb_id=u'abc123', node_id=u'node1'),
CLBNotFoundError(lb_id=u'abc123'),
APIError(code=400, body="", headers={}))
for exception in terminal:
self.assertEqual(
perform_sequence([(eff.intent, lambda i: raise_(exception))],
eff),
(StepResult.FAILURE, [ANY]))
def test_delete_and_verify_verify_404(self):
"""
:func:`delete_and_verify` gets server details after successful delete
and succeeds if get server details returns 404
"""
eff = delete_and_verify('sid')
eff = resolve_effect(
eff, service_request_error_response(APIError(204, {})),
is_error=True)
r = resolve_effect(eff, (StubResponse(404, {}), {"itemNotFound": {}}))
self.assertIsNone(r)
def test_retries_times_out(self):
"""
`RetryingAuthenticator` retries internal authenticator and times out if it
keeps failing for certain period of time
"""
self.mock_auth.authenticate_tenant.side_effect = lambda *a, **kw: fail(
APIError(500, '2'))
d = self.authenticator.authenticate_tenant(23)
self.assertNoResult(d)
self.clock.pump([4] * 4)
f = self.failureResultOf(d, APIError)
self.assertEqual(f.value.code, 500)
def test_create_server_terminal_failures(self):
"""
:obj:`CreateServer.as_effect`, when it results in
:class:`CreateServerConfigurationError` or
:class:`CreateServerOverQuoteError` or a :class:`APIError` with
a 400 failure code, returns with :obj:`StepResult.FAILURE`
"""
errs = (
CreateServerConfigurationError(
"Bad networks format: network uuid is not in proper format "
"(2b55377-890e-4fc9-9ece-ad5a414a788e)"),
CreateServerConfigurationError("This was just a bad request"),
CreateServerOverQuoteError(
"Quota exceeded for ram: Requested 1024, but already used "
"131072 of 131072 ram"),
APIError(code=400, body="Unparsable user error", headers={}),
APIError(code=418, body="I am a teapot but this is still a 4xx",
headers={})
)
self._assert_create_server_with_errs_has_status(
errs, StepResult.FAILURE)
def test_add_nodes_to_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or
if there is any other 4xx error, then
the error is propagated up and the result is a failure.
"""
terminals = (CLBNotFoundError(lb_id=u"12345"),
CLBDeletedError(lb_id=u"12345"),
NoSuchCLBError(lb_id=u"12345"),
CLBNodeLimitError(lb_id=u"12345", node_limit=25),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self._add_one_node_to_clb()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_create_server_retryable_failures(self):
"""
:obj:`CreateServer.as_effect`, when it results in a
:class:`NovaComputeFaultError` or :class:`NovaRateLimitError` or
:class:`APIError` that is not a 4xx, returns with a
:obj:`StepResult.RETRY`
"""
errs = (
NovaComputeFaultError("oops"),
NovaRateLimitError("OverLimit Retry..."),
APIError(code=501, body=":(", headers={}),
TypeError("You did something wrong")
)
self._assert_create_server_with_errs_has_status(errs, StepResult.RETRY)
