def _perform_add_event(self, response_sequence): """ Given a sequence of functions that take an intent and returns a response (or raises an exception), perform :func:`add_event` and return the result. """ log = object() eff = add_event(self.event, 'tid', 'ord', log) uid = '00000000-0000-0000-0000-000000000000' svrq = service_request( ServiceType.CLOUD_FEEDS, 'POST', 'autoscale/events', headers={'content-type': ['application/vnd.rackspace.atom+json']}, data=self._get_request('INFO', uid, 'tid'), log=log, success_pred=has_code(201), json_response=False) seq = [ (TenantScope(mock.ANY, 'tid'), nested_sequence([ retry_sequence( Retry(effect=svrq, should_retry=ShouldDelayAndRetry( can_retry=mock.ANY, next_interval=exponential_backoff_interval(2))), response_sequence) ])) ] return perform_sequence(seq, eff)
def collect_metrics(reactor, config, log, client=None, authenticator=None, _print=False): """ Start collecting the metrics :param reactor: Twisted reactor :param dict config: Configuration got from file containing all info needed to collect metrics :param :class:`silverberg.client.CQLClient` client: Optional cassandra client. A new client will be created if this is not given and disconnected before returing :param :class:`otter.auth.IAuthenticator` authenticator: Optional authenticator. A new authenticator will be created if this is not given :param bool _print: Should debug messages be printed to stdout? :return: :class:`Deferred` fired with ``list`` of `GroupMetrics` """ _client = client or connect_cass_servers(reactor, config['cassandra']) authenticator = authenticator or generate_authenticator(reactor, config['identity']) store = CassScalingGroupCollection(_client, reactor, 1000) dispatcher = get_dispatcher(reactor, authenticator, log, get_service_configs(config), store) # calculate metrics on launch_server and non-paused groups groups = yield perform(dispatcher, Effect(GetAllValidGroups())) groups = [ g for g in groups if json.loads(g["launch_config"]).get("type") == "launch_server" and (not g.get("paused", False))] tenanted_groups = groupby(lambda g: g["tenantId"], groups) group_metrics = yield get_all_metrics( dispatcher, tenanted_groups, log, _print=_print) # Add to cloud metrics metr_conf = config.get("metrics", None) if metr_conf is not None: eff = add_to_cloud_metrics( metr_conf['ttl'], config['region'], group_metrics, len(tenanted_groups), config, log, _print) eff = Effect(TenantScope(eff, metr_conf['tenant_id'])) yield perform(dispatcher, eff) log.msg('added to cloud metrics') if _print: print('added to cloud metrics') if _print: group_metrics.sort(key=lambda g: abs(g.desired - g.actual), reverse=True) print('groups sorted as per divergence') print('\n'.join(map(str, group_metrics))) # Disconnect only if we created the client if not client: yield _client.disconnect() defer.returnValue(group_metrics)
def test_tenant_scope(self): """The :obj:`TenantScope` performer passes through to child effects.""" # This is not testing much, but at least that it calls # perform_tenant_scope in a vaguely working manner. There are # more specific TenantScope performer tests in otter.test.test_http dispatcher = get_full_dispatcher(*([None] * 8)) scope = TenantScope(Effect(Constant('foo')), 1) eff = Effect(scope) self.assertEqual(sync_perform(dispatcher, eff), 'foo')
def convergence_remove_server_from_group(log, transaction_id, server_id, replace, purge, group, state): """ Remove a specific server from the group, optionally decrementing the desired capacity. The server may just be scheduled for deletion, or it may be evicted from the group by removing otter-specific metdata from the server. :param log: A bound logger :param bytes trans_id: The transaction id for this operation. :param bytes server_id: The id of the server to be removed. :param bool replace: Should the server be replaced? :param bool purge: Should the server be deleted from Nova? :param group: The scaling group to remove a server from. :type group: :class:`~otter.models.interface.IScalingGroup` :param state: The current state of the group. :type state: :class:`~otter.models.interface.GroupState` :return: The updated state. :rtype: Effect of :class:`~otter.models.interface.GroupState` :raise: :class:`CannotDeleteServerBelowMinError` if the server cannot be deleted without replacement, and :class:`ServerNotFoundError` if there is no such server to be deleted. """ effects = [_is_server_in_group(group, server_id)] if not replace: effects.append(_can_scale_down(group, server_id)) # the (possibly) two checks can happen in parallel, but we want # ServerNotFoundError to take precedence over # CannotDeleteServerBelowMinError both_checks = yield parallel_all_errors(effects) for is_error, result in both_checks: if is_error: reraise(*result) # Remove the server if purge: eff = set_nova_metadata_item(server_id, *DRAINING_METADATA) else: eff = Effect( EvictServerFromScalingGroup(log=log, transaction_id=transaction_id, scaling_group=group, server_id=server_id)) yield Effect( TenantScope( retry_effect(eff, retry_times(3), exponential_backoff_interval(2)), group.tenant_id)) if not replace: yield do_return(assoc_obj(state, desired=state.desired - 1)) else: yield do_return(state)
def test_perform_service_request(self): """ Performing a :obj:`TenantScope` when it contains a :obj:`ServiceRequest` concretizes the :obj:`ServiceRequest` into a :obj:`Request` as per :func:`concretize_service_request`. """ ereq = service_request(ServiceType.CLOUD_SERVERS, 'GET', 'servers') tscope = TenantScope(ereq, 1) self.assertEqual( sync_perform(self.dispatcher, Effect(tscope)), ('concretized', self.authenticator, self.log, self.service_configs, self.throttler, 1, ereq.intent))
def test_performs_tenant_scope(self, deferred_lock_run): """ :func:`perform_tenant_scope` performs :obj:`TenantScope`, and uses the default throttler """ # We want to ensure # 1. the TenantScope can be performed # 2. the ServiceRequest is run within a lock, since it matches the # default throttling policy set_config_data({ "cloud_client": { "throttling": { "create_server_delay": 1, "delete_server_delay": 0.4 } } }) self.addCleanup(set_config_data, {}) clock = Clock() authenticator = object() log = object() dispatcher = get_cloud_client_dispatcher(clock, authenticator, log, make_service_configs()) svcreq = service_request(ServiceType.CLOUD_SERVERS, 'POST', 'servers') tscope = TenantScope(tenant_id='111', effect=svcreq) def run(f, *args, **kwargs): result = f(*args, **kwargs) result.addCallback(lambda x: (x[0], assoc(x[1], 'locked', True))) return result deferred_lock_run.side_effect = run response = stub_pure_response({}, 200) seq = SequenceDispatcher([ (Authenticate(authenticator=authenticator, tenant_id='111', log=log), lambda i: ('token', fake_service_catalog)), (Request(method='POST', url='http://dfw.openstack/servers', headers=headers('token'), log=log), lambda i: response), ]) disp = ComposedDispatcher([seq, dispatcher]) with seq.consume(): result = perform(disp, Effect(tscope)) self.assertNoResult(result) clock.advance(1) self.assertEqual(self.successResultOf(result), (response[0], { 'locked': True }))
def test_perform_srvreq_nested(self): """ Concretizing of :obj:`ServiceRequest` effects happens even when they are not directly passed as the TenantScope's toplevel Effect, but also when they are returned from callbacks down the line. """ ereq = service_request(ServiceType.CLOUD_SERVERS, 'GET', 'servers') eff = Effect(Constant("foo")).on(lambda r: ereq) tscope = TenantScope(eff, 1) self.assertEqual( sync_perform(self.dispatcher, Effect(tscope)), ('concretized', self.authenticator, self.log, self.service_configs, self.throttler, 1, ereq.intent))
def _generic_rcv3_request(operation, request_bag, lb_id, server_id): """ Perform a generic RCv3 bulk operation on a single (lb, server) pair. :param callable operation: RCv3 function to perform on (lb, server) pair. :param request_bag: An object with a bunch of useful data on it. :param str lb_id: The id of the RCv3 load balancer to act on. :param str server_id: The Nova server id to act on. :return: A deferred that will fire when the request has been performed, firing with the parsed result of the request, or :data:`None` if the request has no body. """ eff = operation(pset([(lb_id, server_id)])) scoped = Effect(TenantScope(eff, request_bag.tenant_id)) return perform(request_bag.dispatcher, scoped)
def group_steps(group): """ Return Effect of list of steps that would be performed on the group if convergence is triggered on it with desired=actual """ now_dt = yield Effect(Func(datetime.utcnow)) all_data_eff = convergence_exec_data(group["tenantId"], group["groupId"], now_dt, get_executor) all_data = yield Effect(TenantScope(all_data_eff, group["tenantId"])) (executor, scaling_group, group_state, desired_group_state, resources) = all_data desired_group_state.desired = len(resources['servers']) steps = executor.plan(desired_group_state, datetime_to_epoch(now_dt), 3600, **resources) yield do_return(steps)
def converge(tenant_id, group_id, dirty_flag): stat = yield Effect(GetStat(dirty_flag)) # If the node disappeared, ignore it. `stat` will be None here if the # divergent flag was discovered only after the group is removed from # currently_converging, but before the divergent flag is deleted, and # then the deletion happens, and then our GetStat happens. This # basically means it happens when one convergence is starting as # another one for the same group is ending. if stat is None: yield msg('converge-divergent-flag-disappeared', znode=dirty_flag) else: eff = converge_one_group(currently_converging, recently_converged, waiting, tenant_id, group_id, stat.version, build_timeout, limited_retry_iterations, step_limits) result = yield Effect(TenantScope(eff, tenant_id)) yield do_return(result)
def add_event(event, admin_tenant_id, region, log): """ Add event to cloud feeds """ event, error, timestamp, event_tenant_id, event_id = sanitize_event(event) req = prepare_request(request_format, event, error, timestamp, region, event_tenant_id, event_id) eff = retry_effect( publish_autoscale_event(req, log=log), compose_retries( lambda f: (not f.check(APIError) or f.value.code < 400 or f.value.code >= 500), retry_times(5)), exponential_backoff_interval(2)) return Effect(TenantScope(tenant_id=admin_tenant_id, effect=eff))
def _is_server_in_group(group, server_id): """ Given a group and server ID, determines if the server is a member of the group. If it isn't, it raises a :class:`ServerNotFoundError`. """ try: response, server_info = yield Effect( TenantScope( retry_effect(get_server_details(server_id), retry_times(3), exponential_backoff_interval(2)), group.tenant_id)) except NoSuchServerError: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id) group_id = group_id_from_metadata( get_in(('server', 'metadata'), server_info, {})) if group_id != group.uuid: raise ServerNotFoundError(group.tenant_id, group.uuid, server_id)
def get_all_metrics_effects(tenanted_groups, log, _print=False): """ Gather server data for and produce metrics for all groups across all tenants in a region :param dict tenanted_groups: Scaling groups grouped with tenantId :param bool _print: Should the function print while processing? :return: ``list`` of :obj:`Effect` of (``list`` of :obj:`GroupMetrics`) or None """ effs = [] for tenant_id, groups in tenanted_groups.iteritems(): eff = get_all_scaling_group_servers() eff = Effect(TenantScope(eff, tenant_id)) eff = eff.on(partial(get_tenant_metrics, tenant_id, groups, _print=_print)) eff = eff.on(list) eff = eff.on( error=lambda exc_info: log.err(exc_info_to_failure(exc_info))) effs.append(eff) return effs
def _generic_rcv3_request(step_class, request_bag, lb_id, server_id): """ Perform a generic RCv3 bulk step on a single (lb, server) pair. :param IStep step_class: The step class to perform the action. :param request_bag: An object with a bunch of useful data on it. :param str lb_id: The id of the RCv3 load balancer to act on. :param str server_id: The Nova server id to act on. :return: A deferred that will fire when the request has been performed, firing with the parsed result of the request, or :data:`None` if the request has no body. """ effect = step_class(lb_node_pairs=s((lb_id, server_id)))._bare_effect() if step_class is BulkAddToRCv3: svc_req = effect.intent codes = set(svc_req.success_pred.codes) - set([409]) svc_req.success_pred = has_code(*codes) # Unfortunate that we have to TenantScope here, but here's where we're # performing. scoped = Effect(TenantScope(effect, request_bag.tenant_id)) d = perform(request_bag.dispatcher, scoped) return d.addCallback(itemgetter(1))
def test_perform_boring(self): """Other effects within a TenantScope are performed as usual.""" tscope = TenantScope(Effect(Constant('foo')), 1) self.assertEqual(sync_perform(self.dispatcher, Effect(tscope)), 'foo')
def dispatcher(self, operation, resp): return SequenceDispatcher([ (TenantScope(mock.ANY, "tid"), nested_sequence([((operation, pset([("lb_id", "server_id")])), lambda i: resp)])) ])
def setUp(self): """ mock dependent functions """ self.connect_cass_servers = patch( self, 'otter.metrics.connect_cass_servers') self.client = mock.Mock(spec=['disconnect']) self.client.disconnect.return_value = succeed(None) self.connect_cass_servers.return_value = self.client self.log = mock_log() self.get_all_metrics = patch(self, 'otter.metrics.get_all_metrics', return_value=succeed("metrics")) self.groups = [{ "tenantId": "t1", "groupId": "g1", "launch_config": '{"type": "launch_server"}' }, { "tenantId": "t1", "groupId": "g2", "launch_config": '{"type": "launch_server"}' }, { "tenantId": "t1", "groupId": "g12", "launch_config": '{"type": "launch_stack"}' }, { "tenantId": "t3", "groupId": "g3", "launch_config": '{"type": "launch_stack"}' }, { "tenantId": "t2", "groupId": "g11", "launch_config": '{"type": "launch_server"}' }] self.lc_groups = {"t1": self.groups[:2], "t2": [self.groups[-1]]} self.add_to_cloud_metrics = patch(self, 'otter.metrics.add_to_cloud_metrics', side_effect=intent_func("atcm")) self.config = { 'cassandra': 'c', 'identity': identity_config, 'metrics': { 'service': 'ms', 'tenant_id': 'tid', 'region': 'IAD', 'ttl': 200, "last_tenant_fpath": "lpath" }, 'region': 'r', 'cloudServersOpenStack': 'nova', 'cloudLoadBalancers': 'clb', 'cloudOrchestration': 'orch', 'rackconnect': 'rc', "non-convergence-tenants": ["ct"] } self.sequence = SequenceDispatcher([ (GetAllValidGroups(), const(self.groups)), (TenantScope(mock.ANY, "tid"), nested_sequence([(("atcm", 200, "r", "metrics", 2, self.config, self.log, False), noop)])) ]) self.get_dispatcher = patch(self, "otter.metrics.get_dispatcher", return_value=self.sequence)
def legacy_intents(): return simple_intents() + [TenantScope(Effect(Constant(None)), 1)]