def test_composition_single_tenant(self): """ authenticator is composed correctly with values from config """ r = mock.Mock() self.config['strategy'] = 'single_tenant' a = generate_authenticator(r, self.config) self.assertIsInstance(a, CachingAuthenticator) self.assertIdentical(a._reactor, r) self.assertEqual(a._ttl, 50) wa = a._authenticator self.assertIsInstance(wa, WaitingAuthenticator) self.assertIdentical(wa._reactor, r) self.assertEqual(wa._wait, 4) ra = wa._authenticator self.assertIsInstance(ra, RetryingAuthenticator) self.assertIdentical(ra._reactor, r) self.assertEqual(ra._max_retries, 3) self.assertEqual(ra._retry_interval, 5) st = ra._authenticator self.assertIsInstance(st, SingleTenantAuthenticator) self.assertEqual(st._identity_user, 'uname') self.assertEqual(st._identity_password, 'pwd') self.assertEqual(st._url, 'htp')
def main(reactor): parser = ArgumentParser( description="Trigger convergence on all/some groups") parser.add_argument( "-c", dest="config", required=True, help="Config file containing identity and cassandra info") parser.add_argument( "-g", nargs="+", dest="group", help=("Group(s) to trigger. Should be in tenantId:groupId form. " "If not provided convergence will be triggerred on all groups " "in CASS")) parser.add_argument("-l", dest="limit", type=int, default=10, help="Concurrency limit. Defaults to 10") parsed = parser.parse_args() conf = json.load(open(parsed.config)) cass_client = connect_cass_servers(reactor, conf["cassandra"]) authenticator = generate_authenticator(reactor, conf["identity"]) store = CassScalingGroupCollection(cass_client, reactor, 1000) if parsed.group: groups = [g.split(":") for g in parsed.group] groups = [{"tenantId": tid, "groupId": gid} for tid, gid in groups] else: groups = yield store.get_all_groups() yield trigger_convergence_groups(authenticator, conf["region"], groups, parsed.limit) yield cass_client.disconnect()
def test_composition_impersonation(self): """ authenticator is composed correctly with values from config """ r = mock.Mock() a = generate_authenticator(r, self.config) self.assertIsInstance(a, CachingAuthenticator) self.assertIdentical(a._reactor, r) self.assertEqual(a._ttl, 50) wa = a._authenticator self.assertIsInstance(wa, WaitingAuthenticator) self.assertIdentical(wa._reactor, r) self.assertEqual(wa._wait, 4) ra = wa._authenticator self.assertIsInstance(ra, RetryingAuthenticator) self.assertIdentical(ra._reactor, r) self.assertEqual(ra._max_retries, 3) self.assertEqual(ra._retry_interval, 5) ia = ra._authenticator self.assertIsInstance(ia, ImpersonatingAuthenticator) self.assertEqual(ia._identity_admin_user, 'uname') self.assertEqual(ia._identity_admin_password, 'pwd') self.assertEqual(ia._url, 'htp') self.assertEqual(ia._admin_url, 'ad')
def __init__(self, reactor, config, log, clock=None, collect=None): """ Initialize the service by connecting to Cassandra and setting up authenticator :param reactor: Twisted reactor for connection purposes :param dict config: All the config necessary to run the service. Comes from config file :param IReactorTime clock: Optional reactor for timer purpose """ self._client = connect_cass_servers(reactor, config['cassandra']) self.log = log self.reactor = reactor self._divergent_groups = {} self.divergent_timeout = get_in( ['metrics', 'divergent_timeout'], config, 3600) self._service = TimerService( get_in(['metrics', 'interval'], config, default=60), collect or self.collect, reactor, config, self.log, client=self._client, authenticator=generate_authenticator(reactor, config['identity'])) self._service.clock = clock or reactor
def main(reactor): parser = ArgumentParser(description="Trigger convergence on all/some groups") parser.add_argument("-c", dest="config", required=True, help="Config file containing identity and cassandra info") group = parser.add_mutually_exclusive_group(required=True) group.add_argument("-g", nargs="+", dest="group", help="Group(s) to trigger. Should be in tenantId:groupId form") group.add_argument("-t", nargs="+", dest="tenant_id", help="TenantID(s) whose group's to trigger") group.add_argument( "--conf-conv-tenants", action="store_true", help=("Convergence triggered on tenants configured as " '"convergence-tenants" setting config file'), ) group.add_argument( "--conf-non-conv-tenants", action="store_true", dest="disabled_tenants", help=("Convergence triggered on all tenants except ones in " '"non-convergence-tenants" setting in conf file'), ) group.add_argument("--all", action="store_true", help="Convergence will be triggered on all groups") parser.add_argument("-l", dest="limit", type=int, default=10, help="Concurrency limit. Defaults to 10") parser.add_argument("--no-error-group", action="store_true", help="Do not converge ERROR groups") parsed = parser.parse_args() conf = json.load(open(parsed.config)) cass_client = connect_cass_servers(reactor, conf["cassandra"]) authenticator = generate_authenticator(reactor, conf["identity"]) store = CassScalingGroupCollection(cass_client, reactor, 1000) groups = yield get_groups(parsed, store, conf) yield trigger_convergence_groups(authenticator, conf["region"], groups, parsed.limit, parsed.no_error_group) yield cass_client.disconnect()
def test_cache_ttl_defaults(self): """ CachingAuthenticator is created with default of 300 if not given """ del self.config['cache_ttl'] r = mock.Mock() a = generate_authenticator(r, self.config) self.assertEqual(a._ttl, 300)
def test_wait_defaults(self): """ WaitingAuthenticator is created with default of 5 if not given """ del self.config['wait'] r = mock.Mock() a = generate_authenticator(r, self.config) self.assertEqual(a._authenticator._wait, 5)
def collect_metrics(reactor, config, log, client=None, authenticator=None, _print=False): """ Start collecting the metrics :param reactor: Twisted reactor :param dict config: Configuration got from file containing all info needed to collect metrics :param :class:`silverberg.client.CQLClient` client: Optional cassandra client. A new client will be created if this is not given and disconnected before returing :param :class:`otter.auth.IAuthenticator` authenticator: Optional authenticator. A new authenticator will be created if this is not given :param bool _print: Should debug messages be printed to stdout? :return: :class:`Deferred` fired with ``list`` of `GroupMetrics` """ _client = client or connect_cass_servers(reactor, config['cassandra']) authenticator = authenticator or generate_authenticator(reactor, config['identity']) store = CassScalingGroupCollection(_client, reactor, 1000) dispatcher = get_dispatcher(reactor, authenticator, log, get_service_configs(config), store) # calculate metrics on launch_server and non-paused groups groups = yield perform(dispatcher, Effect(GetAllValidGroups())) groups = [ g for g in groups if json.loads(g["launch_config"]).get("type") == "launch_server" and (not g.get("paused", False))] tenanted_groups = groupby(lambda g: g["tenantId"], groups) group_metrics = yield get_all_metrics( dispatcher, tenanted_groups, log, _print=_print) # Add to cloud metrics metr_conf = config.get("metrics", None) if metr_conf is not None: eff = add_to_cloud_metrics( metr_conf['ttl'], config['region'], group_metrics, len(tenanted_groups), config, log, _print) eff = Effect(TenantScope(eff, metr_conf['tenant_id'])) yield perform(dispatcher, eff) log.msg('added to cloud metrics') if _print: print('added to cloud metrics') if _print: group_metrics.sort(key=lambda g: abs(g.desired - g.actual), reverse=True) print('groups sorted as per divergence') print('\n'.join(map(str, group_metrics))) # Disconnect only if we created the client if not client: yield _client.disconnect() defer.returnValue(group_metrics)
def collect_metrics(reactor, config, log, client=None, authenticator=None, _print=False): """ Start collecting the metrics :param reactor: Twisted reactor :param dict config: Configuration got from file containing all info needed to collect metrics :param :class:`silverberg.client.CQLClient` client: Optional cassandra client. A new client will be created if this is not given and disconnected before returing :param :class:`otter.auth.IAuthenticator` authenticator: Optional authenticator. A new authenticator will be created if this is not given :param bool _print: Should debug messages be printed to stdout? :return: :class:`Deferred` fired with ``list`` of `GroupMetrics` """ convergence_tids = config.get('convergence-tenants', []) _client = client or connect_cass_servers(reactor, config['cassandra']) authenticator = authenticator or generate_authenticator(reactor, config['identity']) store = CassScalingGroupCollection(_client, reactor, 1000) dispatcher = get_dispatcher(reactor, authenticator, log, get_service_configs(config), store) # calculate metrics fpath = get_in(["metrics", "last_tenant_fpath"], config, default="last_tenant.txt") tenanted_groups = yield perform( dispatcher, get_todays_scaling_groups(convergence_tids, fpath)) group_metrics = yield get_all_metrics( dispatcher, tenanted_groups, log, _print=_print) # Add to cloud metrics metr_conf = config.get("metrics", None) if metr_conf is not None: eff = add_to_cloud_metrics( metr_conf['ttl'], config['region'], group_metrics, len(tenanted_groups), log, _print) eff = Effect(TenantScope(eff, metr_conf['tenant_id'])) yield perform(dispatcher, eff) log.msg('added to cloud metrics') if _print: print('added to cloud metrics') if _print: group_metrics.sort(key=lambda g: abs(g.desired - g.actual), reverse=True) print('groups sorted as per divergence', *group_metrics, sep='\n') # Disconnect only if we created the client if not client: yield _client.disconnect() defer.returnValue(group_metrics)
def main(reactor): conf = json.load(open(sys.argv[1])) conf = conf["default_attributes"]["otter"]["config"] conf["identity"]["strategy"] = "single_tenant" conf["identity"]["username"] = os.environ["OTTERPROD_UNAME"] conf["identity"]["password"] = os.environ["OTTERPROD_PWD"] authenticator = generate_authenticator(reactor, conf["identity"]) token, catalog = yield authenticator.authenticate_tenant("764771") all_tenants = set((yield get_tenant_ids(token, catalog))) worker_tenants = all_tenants - set(conf["convergence-tenants"]) print(*worker_tenants, sep='\n')
def main(reactor): parser = ArgumentParser( description="Trigger convergence on all/some groups") parser.add_argument( "-c", dest="config", required=True, help="Config file containing identity and cassandra info") parser.add_argument( "--steps", action="store_true", help=("Return steps that would be taken if convergence was triggered " "with desired set to current actual. No convergence triggered")) parser.add_argument( "--set-desired-to-actual", action="store_true", dest="set_desired", help="Set group's desired to current actual number of servers") parser.add_argument( "--pause", action="store_true", dest="pause_group", help="Pause given groups") group = parser.add_mutually_exclusive_group(required=True) group.add_argument( "-g", nargs="+", dest="group", help="Group(s) to trigger. Should be in tenantId:groupId form") group.add_argument( "-t", nargs="+", dest="tenant_id", help="TenantID(s) whose group's to trigger") group.add_argument( "--conf-conv-tenants", action="store_true", help=("Convergence triggered on tenants configured as " "\"convergence-tenants\" setting config file")) group.add_argument( "--conf-non-conv-tenants", action="store_true", dest="disabled_tenants", help=("Convergence triggered on all tenants except ones in " "\"non-convergence-tenants\" setting in conf file")) group.add_argument("--all", action="store_true", help="Convergence will be triggered on all groups") parser.add_argument("-l", dest="limit", type=int, default=10, help="Concurrency limit. Defaults to 10") parsed = parser.parse_args() conf = json.load(open(parsed.config)) set_config_data(conf) cass_client = connect_cass_servers(reactor, conf["cassandra"]) authenticator = generate_authenticator(reactor, conf["identity"]) store = CassScalingGroupCollection(cass_client, reactor, 1000) groups = yield get_groups(parsed, store, conf) if parsed.steps: steps = yield groups_steps(groups, reactor, store, cass_client, authenticator, conf) pprint(steps) elif parsed.set_desired: yield set_desired_to_actual(groups, reactor, store, cass_client, authenticator, conf) elif parsed.pause_group: yield pause_groups(cass_client, groups) else: error_groups = yield trigger_convergence_groups( authenticator, conf["region"], groups, parsed.limit) if error_groups: print("Following groups errored") pprint(error_groups) yield cass_client.disconnect()
def makeService(config): """ Set up the otter-api service. """ config = dict(config) set_config_data(config) parent = MultiService() region = config_value('region') seed_endpoints = [ clientFromString(reactor, str(host)) for host in config_value('cassandra.seed_hosts')] cassandra_cluster = LoggingCQLClient( TimingOutCQLClient( reactor, RoundRobinCassandraCluster( seed_endpoints, config_value('cassandra.keyspace'), disconnect_on_cancel=True), config_value('cassandra.timeout') or 30), log.bind(system='otter.silverberg')) store = CassScalingGroupCollection( cassandra_cluster, reactor, config_value('limits.absolute.maxGroups')) admin_store = CassAdmin(cassandra_cluster) bobby_url = config_value('bobby_url') if bobby_url is not None: set_bobby(BobbyClient(bobby_url)) service_configs = get_service_configs(config) authenticator = generate_authenticator(reactor, config['identity']) supervisor = SupervisorService(authenticator, region, coiterate, service_configs) supervisor.setServiceParent(parent) set_supervisor(supervisor) health_checker = HealthChecker(reactor, { 'store': getattr(store, 'health_check', None), 'kazoo': store.kazoo_health_check, 'supervisor': supervisor.health_check }) # Setup cassandra cluster to disconnect when otter shuts down if 'cassandra_cluster' in locals(): parent.addService(FunctionalService(stop=partial( call_after_supervisor, cassandra_cluster.disconnect, supervisor))) otter = Otter(store, region, health_checker.health_check) site = Site(otter.app.resource()) site.displayTracebacks = False api_service = service(str(config_value('port')), site) api_service.setServiceParent(parent) # Setup admin service admin_port = config_value('admin') if admin_port: admin = OtterAdmin(admin_store) admin_site = Site(admin.app.resource()) admin_site.displayTracebacks = False admin_service = service(str(admin_port), admin_site) admin_service.setServiceParent(parent) # setup cloud feed cf_conf = config.get('cloudfeeds', None) if cf_conf is not None: id_conf = deepcopy(config['identity']) id_conf['strategy'] = 'single_tenant' add_to_fanout(CloudFeedsObserver( reactor=reactor, authenticator=generate_authenticator(reactor, id_conf), tenant_id=cf_conf['tenant_id'], region=region, service_configs=service_configs)) # Setup Kazoo client if config_value('zookeeper'): threads = config_value('zookeeper.threads') or 10 disable_logs = config_value('zookeeper.no_logs') threadpool = ThreadPool(maxthreads=threads) sync_kz_client = KazooClient( hosts=config_value('zookeeper.hosts'), # Keep trying to connect until the end of time with # max interval of 10 minutes connection_retry=dict(max_tries=-1, max_delay=600), logger=None if disable_logs else TxLogger(log.bind(system='kazoo')) ) kz_client = TxKazooClient(reactor, threadpool, sync_kz_client) # Don't timeout. Keep trying to connect forever d = kz_client.start(timeout=None) def on_client_ready(_): dispatcher = get_full_dispatcher(reactor, authenticator, log, get_service_configs(config), kz_client, store, supervisor, cassandra_cluster) # Setup scheduler service after starting scheduler = setup_scheduler(parent, dispatcher, store, kz_client) health_checker.checks['scheduler'] = scheduler.health_check otter.scheduler = scheduler # Give dispatcher to Otter REST object otter.dispatcher = dispatcher # Set the client after starting # NOTE: There is small amount of time when the start is # not finished and the kz_client is not set in which case # policy execution and group delete will fail store.kz_client = kz_client # Setup kazoo to stop when shutting down parent.addService(FunctionalService( stop=partial(call_after_supervisor, kz_client.stop, supervisor))) setup_converger( parent, kz_client, dispatcher, config_value('converger.interval') or 10, config_value('converger.build_timeout') or 3600, config_value('converger.limited_retry_iterations') or 10, config_value('converger.step_limits') or {}) d.addCallback(on_client_ready) d.addErrback(log.err, 'Could not start TxKazooClient') return parent
def main(reactor): parser = ArgumentParser( description="Trigger convergence on all/some groups") parser.add_argument( "-c", dest="config", required=True, help="Config file containing identity and cassandra info") parser.add_argument( "--steps", action="store_true", help=("Return steps that would be taken if convergence was triggered " "with desired set to current actual. No convergence triggered")) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( "-g", nargs="+", dest="group", help="Group(s) to trigger. Should be in tenantId:groupId form") group.add_argument("-t", nargs="+", dest="tenant_id", help="TenantID(s) whose group's to trigger") group.add_argument("--conf-conv-tenants", action="store_true", help=("Convergence triggered on tenants configured as " "\"convergence-tenants\" setting config file")) group.add_argument( "--conf-non-conv-tenants", action="store_true", dest="disabled_tenants", help=("Convergence triggered on all tenants except ones in " "\"non-convergence-tenants\" setting in conf file")) group.add_argument("--all", action="store_true", help="Convergence will be triggered on all groups") parser.add_argument("-l", dest="limit", type=int, default=10, help="Concurrency limit. Defaults to 10") parser.add_argument("--no-error-group", action="store_true", help="Do not converge ERROR groups") parsed = parser.parse_args() conf = json.load(open(parsed.config)) cass_client = connect_cass_servers(reactor, conf["cassandra"]) authenticator = generate_authenticator(reactor, conf["identity"]) store = CassScalingGroupCollection(cass_client, reactor, 1000) groups = yield get_groups(parsed, store, conf) if parsed.steps: steps = yield groups_steps(groups, reactor, store, cass_client, authenticator, conf) print(*steps, sep='\n') else: yield trigger_convergence_groups(authenticator, conf["region"], groups, parsed.limit, parsed.no_error_group) yield cass_client.disconnect()