def setUp(self):
"""
Mock CQLClient and log instance
"""
self.client = mock.Mock(spec=['execute', 'disconnect'])
self.log = mock.Mock(spec=['msg'])
self.clock = Clock()
self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
class LoggingCQLClientTests(TestCase):
"""
Tests for mod:`silverberg.logger`
"""
def setUp(self):
"""
Mock CQLClient and log instance
"""
self.client = mock.Mock(spec=['execute', 'disconnect'])
self.log = mock.Mock(spec=['msg'])
self.clock = Clock()
self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
def test_client_execute_success(self):
"""
When client.execute succeeds, it time taken and parameters are recorded
and result is returned
"""
def _execute(*args):
self.clock.advance(10)
return defer.succeed('returnvalue')
self.client.execute.side_effect = _execute
result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
self.assertEqual(self.successResultOf(result), 'returnvalue')
self.client.execute.assert_called_once_with('query', {'d1': 1, 'd2': 2}, 7)
self.log.msg.assert_called_once_with('CQL query executed successfully', query='query',
data={'d1': 1, 'd2': 2}, consistency=7, seconds_taken=10)
def test_client_execute_failure(self):
"""
When client.execute fails the time taken, args and failure are recorded. The failure
is then returned
"""
err = ValueError('v')
def _execute(*args):
self.clock.advance(10)
return defer.fail(err)
self.client.execute.side_effect = _execute
result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
self.assertEqual(self.failureResultOf(result).value, err)
self.client.execute.assert_called_once_with('query', {'d1': 1, 'd2': 2}, 7)
self.log.msg.assert_called_once_with('CQL query execution failed', reason=mock.ANY,
query='query', data={'d1': 1, 'd2': 2}, consistency=7,
seconds_taken=10)
_, kwargs = self.log.msg.call_args
self.assertEqual(kwargs['reason'].value, err)
def test_disconnect(self):
"""
logclient.disconnect() calls internal client's disconnect()
"""
self.client.disconnect.return_value = 'result'
d = self.logclient.disconnect()
self.client.disconnect.assert_called_once_with()
self.assertEquals(d, 'result')
class LoggingCQLClientTests(TestCase):
"""
Tests for mod:`silverberg.logger`
"""
def setUp(self):
"""
Mock CQLClient and log instance
"""
self.client = mock.Mock(spec=["execute"])
self.log = mock.Mock(spec=["msg"])
self.clock = Clock()
self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
def test_client_execute_success(self):
"""
When client.execute succeeds, it time taken and parameters are recorded
and result is returned
"""
def _execute(*args):
self.clock.advance(10)
return defer.succeed("returnvalue")
self.client.execute.side_effect = _execute
result = self.logclient.execute("query", {"d1": 1, "d2": 2}, 7)
self.assertEqual(self.successResultOf(result), "returnvalue")
self.client.execute.assert_called_once_with("query", {"d1": 1, "d2": 2}, 7)
self.log.msg.assert_called_once_with(
"CQL query executed successfully", query="query", data={"d1": 1, "d2": 2}, consistency=7, seconds_taken=10
)
def test_client_execute_failure(self):
"""
When client.execute fails the time taken, args and failure are recorded. The failure
is then returned
"""
err = ValueError("v")
def _execute(*args):
self.clock.advance(10)
return defer.fail(err)
self.client.execute.side_effect = _execute
result = self.logclient.execute("query", {"d1": 1, "d2": 2}, 7)
self.assertEqual(self.failureResultOf(result).value, err)
self.client.execute.assert_called_once_with("query", {"d1": 1, "d2": 2}, 7)
self.log.msg.assert_called_once_with(
"CQL query execution failed",
reason=mock.ANY,
query="query",
data={"d1": 1, "d2": 2},
consistency=7,
seconds_taken=10,
)
_, kwargs = self.log.msg.call_args
self.assertEqual(kwargs["reason"].value, err)
def setUp(self):
"""
Mock CQLClient and log instance
"""
self.client = mock.Mock(spec=['execute', 'disconnect'])
self.log = mock.Mock(spec=['msg'])
self.clock = Clock()
self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
def setUp(self):
"""
Mock CQLClient and log instance
"""
self.client = mock.Mock(spec=["execute"])
self.log = mock.Mock(spec=["msg"])
self.clock = Clock()
self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')
]
cassandra_cluster = LoggingCQLClient(
RoundRobinCassandraCluster(seed_endpoints,
config_value('cassandra.keyspace')),
log.bind(system='otter.silverberg'))
set_store(CassScalingGroupCollection(cassandra_cluster))
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler') and not config_value('mock'):
scheduler_service = SchedulerService(
int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')), cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def makeService(config):
"""
Set up the otter-api service.
"""
config = dict(config)
set_config_data(config)
parent = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'),
disconnect_on_cancel=True),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
store = CassScalingGroupCollection(
cassandra_cluster, reactor, config_value('limits.absolute.maxGroups'))
admin_store = CassAdmin(cassandra_cluster)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
service_configs = get_service_configs(config)
authenticator = generate_authenticator(reactor, config['identity'])
supervisor = SupervisorService(authenticator, region, coiterate,
service_configs)
supervisor.setServiceParent(parent)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
parent.addService(FunctionalService(stop=partial(
call_after_supervisor, cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check)
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(parent)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(parent)
# setup cloud feed
cf_conf = config.get('cloudfeeds', None)
if cf_conf is not None:
id_conf = deepcopy(config['identity'])
id_conf['strategy'] = 'single_tenant'
add_to_fanout(CloudFeedsObserver(
reactor=reactor,
authenticator=generate_authenticator(reactor, id_conf),
tenant_id=cf_conf['tenant_id'],
region=region,
service_configs=service_configs))
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
disable_logs = config_value('zookeeper.no_logs')
threadpool = ThreadPool(maxthreads=threads)
sync_kz_client = KazooClient(
hosts=config_value('zookeeper.hosts'),
# Keep trying to connect until the end of time with
# max interval of 10 minutes
connection_retry=dict(max_tries=-1, max_delay=600),
logger=None if disable_logs else TxLogger(log.bind(system='kazoo'))
)
kz_client = TxKazooClient(reactor, threadpool, sync_kz_client)
# Don't timeout. Keep trying to connect forever
d = kz_client.start(timeout=None)
def on_client_ready(_):
dispatcher = get_full_dispatcher(reactor, authenticator, log,
get_service_configs(config),
kz_client, store, supervisor,
cassandra_cluster)
# Setup scheduler service after starting
scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Give dispatcher to Otter REST object
otter.dispatcher = dispatcher
# Set the client after starting
# NOTE: There is small amount of time when the start is
# not finished and the kz_client is not set in which case
# policy execution and group delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
parent.addService(FunctionalService(
stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
setup_converger(
parent, kz_client, dispatcher,
config_value('converger.interval') or 10,
config_value('converger.build_timeout') or 3600,
config_value('converger.limited_retry_iterations') or 10,
config_value('converger.step_limits') or {})
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return parent
class LoggingCQLClientTests(TestCase):
"""
Tests for mod:`silverberg.logger`
"""
def setUp(self):
"""
Mock CQLClient and log instance
"""
self.client = mock.Mock(spec=['execute', 'disconnect'])
self.log = mock.Mock(spec=['msg'])
self.clock = Clock()
self.logclient = LoggingCQLClient(self.client, self.log, self.clock)
def test_client_execute_success(self):
"""
When client.execute succeeds, it time taken and parameters are recorded
and result is returned
"""
def _execute(*args):
self.clock.advance(10)
return defer.succeed('returnvalue')
self.client.execute.side_effect = _execute
result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
self.assertEqual(self.successResultOf(result), 'returnvalue')
self.client.execute.assert_called_once_with('query', {
'd1': 1,
'd2': 2
}, 7)
self.log.msg.assert_called_once_with('CQL query executed successfully',
query='query',
data={
'd1': 1,
'd2': 2
},
consistency=7,
seconds_taken=10)
def test_client_execute_failure(self):
"""
When client.execute fails the time taken, args and failure are recorded. The failure
is then returned
"""
err = ValueError('v')
def _execute(*args):
self.clock.advance(10)
return defer.fail(err)
self.client.execute.side_effect = _execute
result = self.logclient.execute('query', {'d1': 1, 'd2': 2}, 7)
self.assertEqual(self.failureResultOf(result).value, err)
self.client.execute.assert_called_once_with('query', {
'd1': 1,
'd2': 2
}, 7)
self.log.msg.assert_called_once_with('CQL query execution failed',
reason=mock.ANY,
query='query',
data={
'd1': 1,
'd2': 2
},
consistency=7,
seconds_taken=10)
_, kwargs = self.log.msg.call_args
self.assertEqual(kwargs['reason'].value, err)
def test_disconnect(self):
"""
logclient.disconnect() calls internal client's disconnect()
"""
self.client.disconnect.return_value = 'result'
d = self.logclient.disconnect()
self.client.disconnect.assert_called_once_with()
self.assertEquals(d, 'result')