def _perform_add_event(self, response_sequence):
"""
Given a sequence of functions that take an intent and returns a
response (or raises an exception), perform :func:`add_event` and
return the result.
"""
log = object()
eff = add_event(self.event, 'tid', 'ord', log)
uid = '00000000-0000-0000-0000-000000000000'
svrq = service_request(
ServiceType.CLOUD_FEEDS,
'POST',
'autoscale/events',
headers={'content-type': ['application/vnd.rackspace.atom+json']},
data=self._get_request('INFO', uid, 'tid'),
log=log,
success_pred=has_code(201),
json_response=False)
seq = [
(TenantScope(mock.ANY, 'tid'),
nested_sequence([
retry_sequence(
Retry(effect=svrq,
should_retry=ShouldDelayAndRetry(
can_retry=mock.ANY,
next_interval=exponential_backoff_interval(2))),
response_sequence)
]))
]
return perform_sequence(seq, eff)
def collect_metrics(reactor, config, log, client=None, authenticator=None,
_print=False):
"""
Start collecting the metrics
:param reactor: Twisted reactor
:param dict config: Configuration got from file containing all info
needed to collect metrics
:param :class:`silverberg.client.CQLClient` client:
Optional cassandra client. A new client will be created
if this is not given and disconnected before returing
:param :class:`otter.auth.IAuthenticator` authenticator:
Optional authenticator. A new authenticator will be created
if this is not given
:param bool _print: Should debug messages be printed to stdout?
:return: :class:`Deferred` fired with ``list`` of `GroupMetrics`
"""
_client = client or connect_cass_servers(reactor, config['cassandra'])
authenticator = authenticator or generate_authenticator(reactor,
config['identity'])
store = CassScalingGroupCollection(_client, reactor, 1000)
dispatcher = get_dispatcher(reactor, authenticator, log,
get_service_configs(config), store)
# calculate metrics on launch_server and non-paused groups
groups = yield perform(dispatcher, Effect(GetAllValidGroups()))
groups = [
g for g in groups
if json.loads(g["launch_config"]).get("type") == "launch_server" and
(not g.get("paused", False))]
tenanted_groups = groupby(lambda g: g["tenantId"], groups)
group_metrics = yield get_all_metrics(
dispatcher, tenanted_groups, log, _print=_print)
# Add to cloud metrics
metr_conf = config.get("metrics", None)
if metr_conf is not None:
eff = add_to_cloud_metrics(
metr_conf['ttl'], config['region'], group_metrics,
len(tenanted_groups), config, log, _print)
eff = Effect(TenantScope(eff, metr_conf['tenant_id']))
yield perform(dispatcher, eff)
log.msg('added to cloud metrics')
if _print:
print('added to cloud metrics')
if _print:
group_metrics.sort(key=lambda g: abs(g.desired - g.actual),
reverse=True)
print('groups sorted as per divergence')
print('\n'.join(map(str, group_metrics)))
# Disconnect only if we created the client
if not client:
yield _client.disconnect()
defer.returnValue(group_metrics)
def test_tenant_scope(self):
"""The :obj:`TenantScope` performer passes through to child effects."""
# This is not testing much, but at least that it calls
# perform_tenant_scope in a vaguely working manner. There are
# more specific TenantScope performer tests in otter.test.test_http
dispatcher = get_full_dispatcher(*([None] * 8))
scope = TenantScope(Effect(Constant('foo')), 1)
eff = Effect(scope)
self.assertEqual(sync_perform(dispatcher, eff), 'foo')
def convergence_remove_server_from_group(log, transaction_id, server_id,
replace, purge, group, state):
"""
Remove a specific server from the group, optionally decrementing the
desired capacity.
The server may just be scheduled for deletion, or it may be evicted from
the group by removing otter-specific metdata from the server.
:param log: A bound logger
:param bytes trans_id: The transaction id for this operation.
:param bytes server_id: The id of the server to be removed.
:param bool replace: Should the server be replaced?
:param bool purge: Should the server be deleted from Nova?
:param group: The scaling group to remove a server from.
:type group: :class:`~otter.models.interface.IScalingGroup`
:param state: The current state of the group.
:type state: :class:`~otter.models.interface.GroupState`
:return: The updated state.
:rtype: Effect of :class:`~otter.models.interface.GroupState`
:raise: :class:`CannotDeleteServerBelowMinError` if the server cannot
be deleted without replacement, and :class:`ServerNotFoundError` if
there is no such server to be deleted.
"""
effects = [_is_server_in_group(group, server_id)]
if not replace:
effects.append(_can_scale_down(group, server_id))
# the (possibly) two checks can happen in parallel, but we want
# ServerNotFoundError to take precedence over
# CannotDeleteServerBelowMinError
both_checks = yield parallel_all_errors(effects)
for is_error, result in both_checks:
if is_error:
reraise(*result)
# Remove the server
if purge:
eff = set_nova_metadata_item(server_id, *DRAINING_METADATA)
else:
eff = Effect(
EvictServerFromScalingGroup(log=log,
transaction_id=transaction_id,
scaling_group=group,
server_id=server_id))
yield Effect(
TenantScope(
retry_effect(eff, retry_times(3), exponential_backoff_interval(2)),
group.tenant_id))
if not replace:
yield do_return(assoc_obj(state, desired=state.desired - 1))
else:
yield do_return(state)
def test_perform_service_request(self):
"""
Performing a :obj:`TenantScope` when it contains a
:obj:`ServiceRequest` concretizes the :obj:`ServiceRequest` into a
:obj:`Request` as per :func:`concretize_service_request`.
"""
ereq = service_request(ServiceType.CLOUD_SERVERS, 'GET', 'servers')
tscope = TenantScope(ereq, 1)
self.assertEqual(
sync_perform(self.dispatcher, Effect(tscope)),
('concretized', self.authenticator, self.log, self.service_configs,
self.throttler, 1, ereq.intent))
def test_performs_tenant_scope(self, deferred_lock_run):
"""
:func:`perform_tenant_scope` performs :obj:`TenantScope`, and uses the
default throttler
"""
# We want to ensure
# 1. the TenantScope can be performed
# 2. the ServiceRequest is run within a lock, since it matches the
# default throttling policy
set_config_data({
"cloud_client": {
"throttling": {
"create_server_delay": 1,
"delete_server_delay": 0.4
}
}
})
self.addCleanup(set_config_data, {})
clock = Clock()
authenticator = object()
log = object()
dispatcher = get_cloud_client_dispatcher(clock, authenticator, log,
make_service_configs())
svcreq = service_request(ServiceType.CLOUD_SERVERS, 'POST', 'servers')
tscope = TenantScope(tenant_id='111', effect=svcreq)
def run(f, *args, **kwargs):
result = f(*args, **kwargs)
result.addCallback(lambda x: (x[0], assoc(x[1], 'locked', True)))
return result
deferred_lock_run.side_effect = run
response = stub_pure_response({}, 200)
seq = SequenceDispatcher([
(Authenticate(authenticator=authenticator,
tenant_id='111',
log=log), lambda i: ('token', fake_service_catalog)),
(Request(method='POST',
url='http://dfw.openstack/servers',
headers=headers('token'),
log=log), lambda i: response),
])
disp = ComposedDispatcher([seq, dispatcher])
with seq.consume():
result = perform(disp, Effect(tscope))
self.assertNoResult(result)
clock.advance(1)
self.assertEqual(self.successResultOf(result), (response[0], {
'locked': True
}))
def test_perform_srvreq_nested(self):
"""
Concretizing of :obj:`ServiceRequest` effects happens even when they
are not directly passed as the TenantScope's toplevel Effect, but also
when they are returned from callbacks down the line.
"""
ereq = service_request(ServiceType.CLOUD_SERVERS, 'GET', 'servers')
eff = Effect(Constant("foo")).on(lambda r: ereq)
tscope = TenantScope(eff, 1)
self.assertEqual(
sync_perform(self.dispatcher, Effect(tscope)),
('concretized', self.authenticator, self.log, self.service_configs,
self.throttler, 1, ereq.intent))
def _generic_rcv3_request(operation, request_bag, lb_id, server_id):
"""
Perform a generic RCv3 bulk operation on a single (lb, server) pair.
:param callable operation: RCv3 function to perform on (lb, server) pair.
:param request_bag: An object with a bunch of useful data on it.
:param str lb_id: The id of the RCv3 load balancer to act on.
:param str server_id: The Nova server id to act on.
:return: A deferred that will fire when the request has been performed,
firing with the parsed result of the request, or :data:`None` if the
request has no body.
"""
eff = operation(pset([(lb_id, server_id)]))
scoped = Effect(TenantScope(eff, request_bag.tenant_id))
return perform(request_bag.dispatcher, scoped)
def group_steps(group):
"""
Return Effect of list of steps that would be performed on the group
if convergence is triggered on it with desired=actual
"""
now_dt = yield Effect(Func(datetime.utcnow))
all_data_eff = convergence_exec_data(group["tenantId"], group["groupId"],
now_dt, get_executor)
all_data = yield Effect(TenantScope(all_data_eff, group["tenantId"]))
(executor, scaling_group, group_state, desired_group_state,
resources) = all_data
desired_group_state.desired = len(resources['servers'])
steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt), 3600,
**resources)
yield do_return(steps)
def converge(tenant_id, group_id, dirty_flag):
stat = yield Effect(GetStat(dirty_flag))
# If the node disappeared, ignore it. `stat` will be None here if the
# divergent flag was discovered only after the group is removed from
# currently_converging, but before the divergent flag is deleted, and
# then the deletion happens, and then our GetStat happens. This
# basically means it happens when one convergence is starting as
# another one for the same group is ending.
if stat is None:
yield msg('converge-divergent-flag-disappeared', znode=dirty_flag)
else:
eff = converge_one_group(currently_converging, recently_converged,
waiting, tenant_id, group_id,
stat.version, build_timeout,
limited_retry_iterations, step_limits)
result = yield Effect(TenantScope(eff, tenant_id))
yield do_return(result)
def add_event(event, admin_tenant_id, region, log):
"""
Add event to cloud feeds
"""
event, error, timestamp, event_tenant_id, event_id = sanitize_event(event)
req = prepare_request(request_format, event, error, timestamp, region,
event_tenant_id, event_id)
eff = retry_effect(
publish_autoscale_event(req, log=log),
compose_retries(
lambda f: (not f.check(APIError) or
f.value.code < 400 or
f.value.code >= 500),
retry_times(5)),
exponential_backoff_interval(2))
return Effect(TenantScope(tenant_id=admin_tenant_id, effect=eff))
def _is_server_in_group(group, server_id):
"""
Given a group and server ID, determines if the server is a member of
the group. If it isn't, it raises a :class:`ServerNotFoundError`.
"""
try:
response, server_info = yield Effect(
TenantScope(
retry_effect(get_server_details(server_id), retry_times(3),
exponential_backoff_interval(2)),
group.tenant_id))
except NoSuchServerError:
raise ServerNotFoundError(group.tenant_id, group.uuid, server_id)
group_id = group_id_from_metadata(
get_in(('server', 'metadata'), server_info, {}))
if group_id != group.uuid:
raise ServerNotFoundError(group.tenant_id, group.uuid, server_id)
def get_all_metrics_effects(tenanted_groups, log, _print=False):
"""
Gather server data for and produce metrics for all groups
across all tenants in a region
:param dict tenanted_groups: Scaling groups grouped with tenantId
:param bool _print: Should the function print while processing?
:return: ``list`` of :obj:`Effect` of (``list`` of :obj:`GroupMetrics`)
or None
"""
effs = []
for tenant_id, groups in tenanted_groups.iteritems():
eff = get_all_scaling_group_servers()
eff = Effect(TenantScope(eff, tenant_id))
eff = eff.on(partial(get_tenant_metrics, tenant_id, groups,
_print=_print))
eff = eff.on(list)
eff = eff.on(
error=lambda exc_info: log.err(exc_info_to_failure(exc_info)))
effs.append(eff)
return effs
def _generic_rcv3_request(step_class, request_bag, lb_id, server_id):
"""
Perform a generic RCv3 bulk step on a single (lb, server) pair.
:param IStep step_class: The step class to perform the action.
:param request_bag: An object with a bunch of useful data on it.
:param str lb_id: The id of the RCv3 load balancer to act on.
:param str server_id: The Nova server id to act on.
:return: A deferred that will fire when the request has been performed,
firing with the parsed result of the request, or :data:`None` if the
request has no body.
"""
effect = step_class(lb_node_pairs=s((lb_id, server_id)))._bare_effect()
if step_class is BulkAddToRCv3:
svc_req = effect.intent
codes = set(svc_req.success_pred.codes) - set([409])
svc_req.success_pred = has_code(*codes)
# Unfortunate that we have to TenantScope here, but here's where we're
# performing.
scoped = Effect(TenantScope(effect, request_bag.tenant_id))
d = perform(request_bag.dispatcher, scoped)
return d.addCallback(itemgetter(1))
def test_perform_boring(self):
"""Other effects within a TenantScope are performed as usual."""
tscope = TenantScope(Effect(Constant('foo')), 1)
self.assertEqual(sync_perform(self.dispatcher, Effect(tscope)), 'foo')
def dispatcher(self, operation, resp):
return SequenceDispatcher([
(TenantScope(mock.ANY, "tid"),
nested_sequence([((operation, pset([("lb_id", "server_id")])),
lambda i: resp)]))
])
def setUp(self):
"""
mock dependent functions
"""
self.connect_cass_servers = patch(
self, 'otter.metrics.connect_cass_servers')
self.client = mock.Mock(spec=['disconnect'])
self.client.disconnect.return_value = succeed(None)
self.connect_cass_servers.return_value = self.client
self.log = mock_log()
self.get_all_metrics = patch(self,
'otter.metrics.get_all_metrics',
return_value=succeed("metrics"))
self.groups = [{
"tenantId": "t1",
"groupId": "g1",
"launch_config": '{"type": "launch_server"}'
}, {
"tenantId": "t1",
"groupId": "g2",
"launch_config": '{"type": "launch_server"}'
}, {
"tenantId": "t1",
"groupId": "g12",
"launch_config": '{"type": "launch_stack"}'
}, {
"tenantId": "t3",
"groupId": "g3",
"launch_config": '{"type": "launch_stack"}'
}, {
"tenantId": "t2",
"groupId": "g11",
"launch_config": '{"type": "launch_server"}'
}]
self.lc_groups = {"t1": self.groups[:2], "t2": [self.groups[-1]]}
self.add_to_cloud_metrics = patch(self,
'otter.metrics.add_to_cloud_metrics',
side_effect=intent_func("atcm"))
self.config = {
'cassandra': 'c',
'identity': identity_config,
'metrics': {
'service': 'ms',
'tenant_id': 'tid',
'region': 'IAD',
'ttl': 200,
"last_tenant_fpath": "lpath"
},
'region': 'r',
'cloudServersOpenStack': 'nova',
'cloudLoadBalancers': 'clb',
'cloudOrchestration': 'orch',
'rackconnect': 'rc',
"non-convergence-tenants": ["ct"]
}
self.sequence = SequenceDispatcher([
(GetAllValidGroups(), const(self.groups)),
(TenantScope(mock.ANY, "tid"),
nested_sequence([(("atcm", 200, "r", "metrics", 2, self.config,
self.log, False), noop)]))
])
self.get_dispatcher = patch(self,
"otter.metrics.get_dispatcher",
return_value=self.sequence)
def legacy_intents():
return simple_intents() + [TenantScope(Effect(Constant(None)), 1)]