def connect_cass_servers(reactor, config): """ Connect to Cassandra servers and return the connection """ seed_endpoints = [clientFromString(reactor, str(host)) for host in config['seed_hosts']] return RoundRobinCassandraCluster( seed_endpoints, config['keyspace'], disconnect_on_cancel=True)
def makeService(config): """ Set up the otter-api service. """ set_config_data(dict(config)) if not config_value('mock'): seed_endpoints = [ clientFromString(reactor, str(host)) for host in config_value('cassandra.seed_hosts') ] cassandra_cluster = LoggingCQLClient( RoundRobinCassandraCluster(seed_endpoints, config_value('cassandra.keyspace')), log.bind(system='otter.silverberg')) set_store(CassScalingGroupCollection(cassandra_cluster)) bobby_url = config_value('bobby_url') if bobby_url is not None: set_bobby(BobbyClient(bobby_url)) cache_ttl = config_value('identity.cache_ttl') if cache_ttl is None: # FIXME: Pick an arbitrary cache ttl value based on absolutely no # science. cache_ttl = 300 authenticator = CachingAuthenticator( reactor, ImpersonatingAuthenticator(config_value('identity.username'), config_value('identity.password'), config_value('identity.url'), config_value('identity.admin_url')), cache_ttl) supervisor = Supervisor(authenticator.authenticate_tenant, coiterate) set_supervisor(supervisor) s = MultiService() site = Site(root) site.displayTracebacks = False api_service = service(str(config_value('port')), site) api_service.setServiceParent(s) if config_value('scheduler') and not config_value('mock'): scheduler_service = SchedulerService( int(config_value('scheduler.batchsize')), int(config_value('scheduler.interval')), cassandra_cluster) scheduler_service.setServiceParent(s) return s
def test_round_robin_execute(self): """ seed_clients are chosen in round robin manner """ cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') for client, arg in [(1, 'foo'), (2, 'bar'), (0, 'baz'), (1, 'bax')]: result = cluster.execute(arg) self.clients[client].execute.assert_called_with(arg) self.assertEqual(self.clients[client].execute.return_value, result)
def test_other_error_propogated(self): """ Any error other than subclass of ``ConnectError`` is propgoated """ cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') rand_err = ValueError('random err') self.clients[1].execute.return_value = defer.fail(rand_err) result = cluster.execute(2, 3) self.assertEqual(self.failureResultOf(result).value, rand_err) self.assertEqual(self.clients[2].execute.called, False) self.assertEqual(self.clients[0].execute.called, False)
def test_one_node_down(self): """ If a cass node is down, it tries the next node in cluster """ cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') self.clients[1].execute.return_value = defer.fail(ConnectionRefusedError()) result = cluster.execute(2, 3) self.assertEqual(self.successResultOf(result), 'exec_ret3') self.clients[1].execute.assert_called_once_with(2, 3) self.clients[2].execute.assert_called_once_with(2, 3) self.assertFalse(self.clients[0].execute.called)
def test_other_error_propogated_on_node_down(self): """ If first node gives ``ConnectError`` then second node is tried if it gives error other than subclass of ``ConnectError`` it is propogated """ cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') self.clients[1].execute.return_value = defer.fail(NoRouteError()) rand_err = ValueError('random err') self.clients[2].execute.return_value = defer.fail(rand_err) result = cluster.execute(2, 3) self.assertEqual(self.failureResultOf(result).value, rand_err) self.assertEqual(self.clients[0].execute.called, False)
def test_all_nodes_down(self): """ If all cass nodes are down, it gives up eventually by raising the connection error exception """ cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') err = ConnectionRefusedError() for i in range(3): self.clients[i].execute.side_effect = lambda *_: defer.fail(err) result = cluster.execute(2, 3) self.assertEqual(self.failureResultOf(result).value, err) for i in range(3): self.clients[i].execute.assert_called_once_with(2, 3)
def test_multiple_clients(self): """ When multiple clients execute simultaneously, it does not skip nodes. It basically ensures that http://bit.ly/1csEIRR situation does not happen """ cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') def _execute(*args): cluster.execute(2, 3) return defer.fail(NoRouteError()) self.clients[1].execute.side_effect = _execute result = cluster.execute(2, 3) self.assertEqual(self.successResultOf(result), 'exec_ret3') self.assertEqual(self.clients[2].execute.call_args_list, [mock.call(2, 3)] * 2) self.clients[1].execute.assert_called_once_with(2, 3)
def makeService(config): """ Set up the otter-api service. """ config = dict(config) set_config_data(config) parent = MultiService() region = config_value('region') seed_endpoints = [ clientFromString(reactor, str(host)) for host in config_value('cassandra.seed_hosts')] cassandra_cluster = LoggingCQLClient( TimingOutCQLClient( reactor, RoundRobinCassandraCluster( seed_endpoints, config_value('cassandra.keyspace'), disconnect_on_cancel=True), config_value('cassandra.timeout') or 30), log.bind(system='otter.silverberg')) store = CassScalingGroupCollection( cassandra_cluster, reactor, config_value('limits.absolute.maxGroups')) admin_store = CassAdmin(cassandra_cluster) bobby_url = config_value('bobby_url') if bobby_url is not None: set_bobby(BobbyClient(bobby_url)) service_configs = get_service_configs(config) authenticator = generate_authenticator(reactor, config['identity']) supervisor = SupervisorService(authenticator, region, coiterate, service_configs) supervisor.setServiceParent(parent) set_supervisor(supervisor) health_checker = HealthChecker(reactor, { 'store': getattr(store, 'health_check', None), 'kazoo': store.kazoo_health_check, 'supervisor': supervisor.health_check }) # Setup cassandra cluster to disconnect when otter shuts down if 'cassandra_cluster' in locals(): parent.addService(FunctionalService(stop=partial( call_after_supervisor, cassandra_cluster.disconnect, supervisor))) otter = Otter(store, region, health_checker.health_check) site = Site(otter.app.resource()) site.displayTracebacks = False api_service = service(str(config_value('port')), site) api_service.setServiceParent(parent) # Setup admin service admin_port = config_value('admin') if admin_port: admin = OtterAdmin(admin_store) admin_site = Site(admin.app.resource()) admin_site.displayTracebacks = False admin_service = service(str(admin_port), admin_site) admin_service.setServiceParent(parent) # setup cloud feed cf_conf = config.get('cloudfeeds', None) if cf_conf is not None: id_conf = deepcopy(config['identity']) id_conf['strategy'] = 'single_tenant' add_to_fanout(CloudFeedsObserver( reactor=reactor, authenticator=generate_authenticator(reactor, id_conf), tenant_id=cf_conf['tenant_id'], region=region, service_configs=service_configs)) # Setup Kazoo client if config_value('zookeeper'): threads = config_value('zookeeper.threads') or 10 disable_logs = config_value('zookeeper.no_logs') threadpool = ThreadPool(maxthreads=threads) sync_kz_client = KazooClient( hosts=config_value('zookeeper.hosts'), # Keep trying to connect until the end of time with # max interval of 10 minutes connection_retry=dict(max_tries=-1, max_delay=600), logger=None if disable_logs else TxLogger(log.bind(system='kazoo')) ) kz_client = TxKazooClient(reactor, threadpool, sync_kz_client) # Don't timeout. Keep trying to connect forever d = kz_client.start(timeout=None) def on_client_ready(_): dispatcher = get_full_dispatcher(reactor, authenticator, log, get_service_configs(config), kz_client, store, supervisor, cassandra_cluster) # Setup scheduler service after starting scheduler = setup_scheduler(parent, dispatcher, store, kz_client) health_checker.checks['scheduler'] = scheduler.health_check otter.scheduler = scheduler # Give dispatcher to Otter REST object otter.dispatcher = dispatcher # Set the client after starting # NOTE: There is small amount of time when the start is # not finished and the kz_client is not set in which case # policy execution and group delete will fail store.kz_client = kz_client # Setup kazoo to stop when shutting down parent.addService(FunctionalService( stop=partial(call_after_supervisor, kz_client.stop, supervisor))) setup_converger( parent, kz_client, dispatcher, config_value('converger.interval') or 10, config_value('converger.build_timeout') or 3600, config_value('converger.limited_retry_iterations') or 10, config_value('converger.step_limits') or {}) d.addCallback(on_client_ready) d.addErrback(log.err, 'Could not start TxKazooClient') return parent
def test_disconnect(self): cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace') cluster.disconnect() for client in self.clients: client.disconnect.assert_called_with()