def test_throttling(self):
"""
When the throttler function returns a bracketing function, it's used to
throttle the request.
"""
def throttler(stype, method, tid):
if (stype == ServiceType.CLOUD_SERVERS and method == 'get'
and tid == 1):
return bracket
bracket = object()
svcreq = service_request(ServiceType.CLOUD_SERVERS, 'GET',
'servers').intent
response = stub_pure_response({}, 200)
seq = SequenceDispatcher([
(_Throttle(bracket=bracket, effect=mock.ANY),
nested_sequence([
(Authenticate(authenticator=self.authenticator,
tenant_id=1,
log=self.log), lambda i:
('token', fake_service_catalog)),
(Request(method='GET',
url='http://dfw.openstack/servers',
headers=headers('token'),
log=self.log), lambda i: response),
])),
])
eff = self._concrete(svcreq, throttler=throttler)
with seq.consume():
result = sync_perform(seq, eff)
self.assertEqual(result, (response[0], {}))
def test_print(self):
sequence = SequenceDispatcher([
(Print('What... is your quest?'), lambda _: None),
])
with sequence.consume():
sync_perform(sequence, program())
def test_is_acquired_performs(self):
"""
is_acquired performs effect from is_acquired_eff
"""
self.lock.dispatcher = SequenceDispatcher([(("is_acquired", ),
const("ret"))])
self.lock.is_acquired_eff = intent_func("is_acquired")
self.assertEqual(self.successResultOf(self.lock.is_acquired()), "ret")
def test_release_performs(self):
"""
release performs effect from release_eff
"""
self.lock.dispatcher = SequenceDispatcher([(("release", ),
const("ret"))])
self.lock.release_eff = intent_func("release")
self.assertEqual(self.successResultOf(self.lock.release()), "ret")
def test_setup_no_groups(self):
"""
``self.s.setup()`` gets groups and does nothing if there are no groups
"""
self.s.dispatcher = SequenceDispatcher([(("ggtc", "cf"), const([]))])
d = self.s.setup()
self.successResultOf(d)
self.assertEqual(self.s._calls, [])
self.assertEqual(self.clock.getDelayedCalls(), [])
def test_echo(self, line):
sequence = SequenceDispatcher([
(Print('What... is your quest?'), lambda _: None),
(Readline(), lambda _: line),
(Print(line), lambda _: None),
])
with sequence.consume():
sync_perform(sequence, echo())
def test_acquire_performs(self):
"""
acquire performs effect from acquire_eff
"""
self.lock.dispatcher = SequenceDispatcher([
(("acquire", "blocking", "timeout"), const("ret"))
])
self.lock.acquire_eff = intent_func("acquire")
self.assertEqual(
self.successResultOf(self.lock.acquire("blocking", "timeout")),
"ret")
def test_setup_err(self):
"""
``self.s.setup()`` will log any error and return success
"""
self.s.dispatcher = SequenceDispatcher([(("ggtc", "cf"),
conste(ValueError("h")))])
d = self.s.setup()
self.successResultOf(d)
self.log.err.assert_called_once_with(CheckFailure(ValueError),
"selfheal-setup-err",
otter_service="selfheal")
def test_performs_tenant_scope(self, deferred_lock_run):
"""
:func:`perform_tenant_scope` performs :obj:`TenantScope`, and uses the
default throttler
"""
# We want to ensure
# 1. the TenantScope can be performed
# 2. the ServiceRequest is run within a lock, since it matches the
# default throttling policy
set_config_data({
"cloud_client": {
"throttling": {
"create_server_delay": 1,
"delete_server_delay": 0.4
}
}
})
self.addCleanup(set_config_data, {})
clock = Clock()
authenticator = object()
log = object()
dispatcher = get_cloud_client_dispatcher(clock, authenticator, log,
make_service_configs())
svcreq = service_request(ServiceType.CLOUD_SERVERS, 'POST', 'servers')
tscope = TenantScope(tenant_id='111', effect=svcreq)
def run(f, *args, **kwargs):
result = f(*args, **kwargs)
result.addCallback(lambda x: (x[0], assoc(x[1], 'locked', True)))
return result
deferred_lock_run.side_effect = run
response = stub_pure_response({}, 200)
seq = SequenceDispatcher([
(Authenticate(authenticator=authenticator,
tenant_id='111',
log=log), lambda i: ('token', fake_service_catalog)),
(Request(method='POST',
url='http://dfw.openstack/servers',
headers=headers('token'),
log=log), lambda i: response),
])
disp = ComposedDispatcher([seq, dispatcher])
with seq.consume():
result = perform(disp, Effect(tscope))
self.assertNoResult(result)
clock.advance(1)
self.assertEqual(self.successResultOf(result), (response[0], {
'locked': True
}))
def test_without_metrics(self):
"""
Doesnt add metrics to blueflood if metrics config is not there
"""
sequence = SequenceDispatcher([(GetAllValidGroups(),
const(self.groups))])
self.get_dispatcher.return_value = sequence
del self.config["metrics"]
with sequence.consume():
d = collect_metrics("reactor", self.config, self.log)
self.assertEqual(self.successResultOf(d), "metrics")
self.assertFalse(self.add_to_cloud_metrics.called)
def test_sequence():
"""Collects each Effectful result into a list."""
effs = [Effect('a'), Effect('b'), Effect('c')]
dispatcher = SequenceDispatcher([
('a', lambda i: 'Ei'),
('b', lambda i: 'Bee'),
('c', lambda i: 'Cee'),
])
eff = sequence(effs)
with dispatcher.consume():
result = sync_perform(_base_and(dispatcher), eff)
assert result == ['Ei', 'Bee', 'Cee']
def test_challenge(self, line):
sequence = SequenceDispatcher([
(Print('What... is your quest?'), lambda _:None),
(Readline(), lambda _: line),
(Print('What... is your quest?'), lambda _:None),
(Readline(), lambda _:'To seek the Holy Grail.\n'),
(Print('What... is your favourite colour?'), lambda _:None),
])
with sequence.consume():
dispatcher = ComposedDispatcher([
sequence,
base_dispatcher,
])
sync_perform(dispatcher, challenge())
def test_remove_nodes_from_clb_success_failures(self):
"""
:obj:`AddNodesToCLB` succeeds if the CLB is not in existence (has been
deleted or is not found).
"""
successes = [CLBNotFoundError(lb_id=u'12345'),
CLBDeletedError(lb_id=u'12345'),
NoSuchCLBError(lb_id=u'12345')]
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in successes:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(sync_perform(seq, eff),
(StepResult.SUCCESS, []))
def test_add_nodes_to_clb_success_response_codes(self):
"""
:obj:`AddNodesToCLB` succeeds on 202.
"""
eff = self._add_one_node_to_clb()
seq = SequenceDispatcher([
(eff.intent, lambda i: (StubResponse(202, {}), '')),
(Log(ANY, ANY), lambda _: None)
])
expected = (
StepResult.RETRY,
[ErrorReason.String('must re-gather after adding to CLB in order '
'to update the active cache')])
with seq.consume():
self.assertEquals(sync_perform(seq, eff), expected)
def test_setup(self):
"""
``self.s.setup()`` will setup convergences to be triggered over
specified time range
"""
self.s.dispatcher = SequenceDispatcher([(("ggtc", "cf"),
const(self.groups))])
d = self.s.setup()
self.successResultOf(d)
calls = self.clock.getDelayedCalls()
self.assertEqual(self.s._calls, calls)
for i, c in enumerate(calls):
self.assertEqual(c.getTime(), i * 60)
self.assertEqual(c.func, sh.perform)
self.assertEqual(c.args,
(self.s.dispatcher, "t{}g{}".format(i, i)))
def test_fold_effect():
"""
:func:`fold_effect` folds the given function over the results of the
effects.
"""
effs = [Effect('a'), Effect('b'), Effect('c')]
dispatcher = SequenceDispatcher([
('a', lambda i: 'Ei'),
('b', lambda i: 'Bee'),
('c', lambda i: 'Cee'),
])
eff = fold_effect(operator.add, 'Nil', effs)
with dispatcher.consume():
result = sync_perform(_base_and(dispatcher), eff)
assert result == 'NilEiBeeCee'
def test_remove_nodes_from_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if there are any 4xx errors, then
the error is propagated up and the result is a failure.
"""
terminals = (APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_fold_effect_errors():
"""
When one of the effects in the folding list fails, a FoldError is raised
with the accumulator so far.
"""
effs = [Effect('a'), Effect(Error(ZeroDivisionError('foo'))), Effect('c')]
dispatcher = SequenceDispatcher([
('a', lambda i: 'Ei'),
])
eff = fold_effect(operator.add, 'Nil', effs)
with dispatcher.consume():
with raises(FoldError) as excinfo:
sync_perform(_base_and(dispatcher), eff)
assert excinfo.value.accumulator == 'NilEi'
assert excinfo.value.wrapped_exception[0] is ZeroDivisionError
assert str(excinfo.value.wrapped_exception[1]) == 'foo'
def test_sequence_error():
"""
Allows :obj:`FoldError` to be raised when an Effect fails. The list
accumulated so far is the `accumulator` value in the :obj:`FoldError`.
"""
effs = [Effect('a'), Effect(Error(ZeroDivisionError('foo'))), Effect('c')]
dispatcher = SequenceDispatcher([
('a', lambda i: 'Ei'),
])
eff = sequence(effs)
with dispatcher.consume():
with raises(FoldError) as excinfo:
sync_perform(_base_and(dispatcher), eff)
assert excinfo.value.accumulator == ['Ei']
assert excinfo.value.wrapped_exception[0] is ZeroDivisionError
assert str(excinfo.value.wrapped_exception[1]) == 'foo'
def test_remove_nodes_from_clb_non_terminal_failures_to_retry(self):
"""
:obj:`RemoveNodesFromCLB` retries if the CLB is temporarily locked,
or if the request was rate-limited, or if there was an API error and
the error is unknown but not a 4xx.
"""
non_terminals = (CLBImmutableError(lb_id=u"12345"),
CLBRateLimitError(lb_id=u"12345"),
APIError(code=500, body="oops!"),
TypeError("You did something wrong in your code."))
eff = RemoveNodesFromCLB(lb_id='12345',
node_ids=pset(['1', '2'])).as_effect()
for exc in non_terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.RETRY, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_add_nodes_to_clb_terminal_failures(self):
"""
:obj:`AddNodesToCLB` fails if the CLB is not found or deleted, or
if there is any other 4xx error, then
the error is propagated up and the result is a failure.
"""
terminals = (CLBNotFoundError(lb_id=u"12345"),
CLBDeletedError(lb_id=u"12345"),
NoSuchCLBError(lb_id=u"12345"),
CLBNodeLimitError(lb_id=u"12345", node_limit=25),
APIError(code=403, body="You're out of luck."),
APIError(code=422, body="Oh look another 422."))
eff = self._add_one_node_to_clb()
for exc in terminals:
seq = SequenceDispatcher([(eff.intent, lambda i: raise_(exc))])
with seq.consume():
self.assertEquals(
sync_perform(seq, eff),
(StepResult.FAILURE, [ErrorReason.Exception(
matches(ContainsAll([type(exc), exc])))]))
def test_setup_still_active(self):
"""
If there are scheduled calls when perform is called, they are
cancelled and err is logged. Future calls are scheduled as usual
"""
self.clock.advance(-0.6)
call1 = self.clock.callLater(1, noop, None)
call2 = self.clock.callLater(0, noop, None)
call3 = self.clock.callLater(2, noop, None)
self.clock.advance(0.6)
self.s._calls = [call1, call2, call3]
self.s.dispatcher = SequenceDispatcher([(("ggtc", "cf"),
const(self.groups))])
d = self.s.setup()
self.successResultOf(d)
self.log.err.assert_called_once_with(matches(IsInstance(RuntimeError)),
"selfheal-calls-err",
active=2,
otter_service="selfheal")
self.assertFalse(call1.active())
self.assertFalse(call2.active())
def assert_logs(self, steps, intents):
"""Log some steps and ensure they result in the given Log intents."""
sequence = SequenceDispatcher([(intent, noop) for intent in intents])
with sequence.consume():
sync_perform(test_dispatcher(sequence), log_steps(steps))
def dispatcher(self, operation, resp):
return SequenceDispatcher([
(TenantScope(mock.ANY, "tid"),
nested_sequence([((operation, pset([("lb_id", "server_id")])),
lambda i: resp)]))
])
def setUp(self):
"""
mock dependent functions
"""
self.connect_cass_servers = patch(
self, 'otter.metrics.connect_cass_servers')
self.client = mock.Mock(spec=['disconnect'])
self.client.disconnect.return_value = succeed(None)
self.connect_cass_servers.return_value = self.client
self.log = mock_log()
self.get_all_metrics = patch(self,
'otter.metrics.get_all_metrics',
return_value=succeed("metrics"))
self.groups = [{
"tenantId": "t1",
"groupId": "g1",
"launch_config": '{"type": "launch_server"}'
}, {
"tenantId": "t1",
"groupId": "g2",
"launch_config": '{"type": "launch_server"}'
}, {
"tenantId": "t1",
"groupId": "g12",
"launch_config": '{"type": "launch_stack"}'
}, {
"tenantId": "t3",
"groupId": "g3",
"launch_config": '{"type": "launch_stack"}'
}, {
"tenantId": "t2",
"groupId": "g11",
"launch_config": '{"type": "launch_server"}'
}]
self.lc_groups = {"t1": self.groups[:2], "t2": [self.groups[-1]]}
self.add_to_cloud_metrics = patch(self,
'otter.metrics.add_to_cloud_metrics',
side_effect=intent_func("atcm"))
self.config = {
'cassandra': 'c',
'identity': identity_config,
'metrics': {
'service': 'ms',
'tenant_id': 'tid',
'region': 'IAD',
'ttl': 200,
"last_tenant_fpath": "lpath"
},
'region': 'r',
'cloudServersOpenStack': 'nova',
'cloudLoadBalancers': 'clb',
'cloudOrchestration': 'orch',
'rackconnect': 'rc',
"non-convergence-tenants": ["ct"]
}
self.sequence = SequenceDispatcher([
(GetAllValidGroups(), const(self.groups)),
(TenantScope(mock.ANY, "tid"),
nested_sequence([(("atcm", 200, "r", "metrics", 2, self.config,
self.log, False), noop)]))
])
self.get_dispatcher = patch(self,
"otter.metrics.get_dispatcher",
return_value=self.sequence)
