def setUp(self):
"""
Setup fake supervisor and fake job
"""
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.log = mock_log()
self.jobs = []
class FakeJob(object):
def __init__(jself, *args):
jself.args = args
jself.job_id = len(self.jobs) + 10
self.jobs.append(jself)
def start(jself, launch):
jself.launch = launch
jself.d = Deferred()
return jself.d
patch(self, "otter.supervisor._Job", new=FakeJob)
self.state = GroupState("t", "g", "n", {}, {}, 0, 1, 2, ScalingGroupStatus.ACTIVE)
def setUp(self):
"""
Replace the store every time with a clean one.
"""
store = MockScalingGroupCollection()
self.root = Otter(store).app.resource()
set_config_data({'url_root': 'http://127.0.0.1',
'limits': {'pagination': 5}})
self.addCleanup(set_config_data, {})
self.config = config()[1]
self.config['minEntities'] = 0
self.active_pending_etc = ({}, {}, 'date', {}, False)
# patch both the config and the groups
self.mock_controller = patch(self, 'otter.rest.configs.controller',
spec=['obey_config_change'])
patch(self, 'otter.rest.groups.controller', new=self.mock_controller)
# Patch supervisor
supervisor = mock.Mock(spec=['validate_launch_config'])
supervisor.validate_launch_config.return_value = defer.succeed(None)
set_supervisor(supervisor)
def _mock_obey_config_change(log, trans, config, group, state):
return defer.succeed(GroupState(
state.tenant_id, state.group_id, state.group_name, *self.active_pending_etc))
self.mock_controller.obey_config_change.side_effect = _mock_obey_config_change
def setUp(self):
"""
Setup fake supervisor and fake job
"""
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.log = mock_log()
self.jobs = []
class FakeJob(object):
def __init__(jself, *args):
jself.args = args
jself.job_id = len(self.jobs) + 10
self.jobs.append(jself)
def start(jself, launch):
jself.launch = launch
jself.d = Deferred()
return jself.d
patch(self, 'otter.supervisor._Job', new=FakeJob)
self.state = GroupState('t', 'g', 'n', {}, {}, *range(3))
def setUp(self):
"""
Set the Cassandra store, and also patch the controller
"""
keyspace.resume()
self.root = Otter(store, 'ord').app.resource()
set_config_data(limits)
self.addCleanup(set_config_data, {})
self.config = config()[0]
self.config['minEntities'] = 0
self.active_pending_etc = (self.config['name'], {}, {}, 'date', {}, False)
# patch both the config and the groups
self.mock_controller = patch(self, 'otter.rest.configs.controller',
spec=['obey_config_change'])
patch(self, 'otter.rest.groups.controller', new=self.mock_controller)
# Patch supervisor
supervisor = mock.Mock(spec=['validate_launch_config'])
supervisor.validate_launch_config.return_value = defer.succeed(None)
set_supervisor(supervisor)
def _mock_obey_config_change(log, trans, config, group, state,
launch_config):
return defer.succeed(GroupState(
state.tenant_id, state.group_id, *self.active_pending_etc))
self.mock_controller.obey_config_change.side_effect = _mock_obey_config_change
store.kz_client = mock.Mock(Lock=self.mock_lock())
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')
]
cassandra_cluster = LoggingCQLClient(
RoundRobinCassandraCluster(seed_endpoints,
config_value('cassandra.keyspace')),
log.bind(system='otter.silverberg'))
set_store(CassScalingGroupCollection(cassandra_cluster))
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler') and not config_value('mock'):
scheduler_service = SchedulerService(
int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')), cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def tearDown(self):
"""
Disconnect the client - it will reconnect as needed. Better if this
could be disconnected only once this particular module were done.
"""
keyspace.dirtied()
keyspace.pause()
keyspace.reset()
set_supervisor(None)
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace')), log.bind(system='otter.silverberg'))
set_store(CassScalingGroupCollection(cassandra_cluster))
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(
config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler') and not config_value('mock'):
scheduler_service = SchedulerService(int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')),
cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.group = iMock(IScalingGroup, tenant_id='tenant', uuid='group')
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, desired=1)
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, desired=1)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, 'otter.supervisor.generate_job_id', return_value='jid')
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
def setUp(self):
"""
Mock modify state
"""
super(AllGroupsEndpointTestCase, self).setUp()
self.mock_controller = patch(self, 'otter.rest.groups.controller')
# Patch supervisor
self.supervisor = mock.Mock(spec=['validate_launch_config'])
self.supervisor.validate_launch_config.return_value = defer.succeed(None)
set_supervisor(self.supervisor)
set_config_data({'limits': {'pagination': 100}, 'url_root': ''})
def setUp(self):
"""
Set up mock Bobby client
"""
set_bobby(BobbyClient("http://127.0.0.1:9876/"))
super(AllGroupsBobbyEndpointTestCase, self).setUp()
self.mock_controller = patch(self, 'otter.rest.groups.controller')
patch(self, 'otter.util.http.get_url_root', return_value="")
# Patch supervisor
supervisor = mock.Mock(spec=['validate_launch_config'])
supervisor.validate_launch_config.return_value = defer.succeed(None)
set_supervisor(supervisor)
def setUp(self):
"""
Set up a mock group to be used for viewing and updating configurations
"""
super(LaunchConfigTestCase, self).setUp()
self.mock_group = mock.MagicMock(
spec=('uuid', 'view_launch_config', 'update_launch_config'),
uuid='1')
self.mock_store.get_scaling_group.return_value = self.mock_group
# Patch supervisor
self.supervisor = mock.Mock(spec=['validate_launch_config'])
self.supervisor.validate_launch_config.return_value = defer.succeed(None)
set_supervisor(self.supervisor)
def setUp(self):
"""
Set up a mock group to be used for viewing and updating configurations
"""
super(LaunchConfigTestCase, self).setUp()
self.mock_group = mock.MagicMock(spec=('uuid', 'view_launch_config',
'update_launch_config'),
uuid='1')
self.mock_store.get_scaling_group.return_value = self.mock_group
# Patch supervisor
self.supervisor = mock.Mock(spec=['validate_launch_config'])
self.supervisor.validate_launch_config.return_value = defer.succeed(
None)
set_supervisor(self.supervisor)
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = "trans_id"
self.log = mock_log()
self.state = GroupState(
"tid", "gid", "g", {"s0": {"id": "s0"}}, {}, None, None, None, ScalingGroupStatus.ACTIVE, desired=1
)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, "otter.supervisor.generate_job_id", return_value="jid")
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.group.view_config.return_value = succeed({"minEntities": 0})
self.group.view_launch_config.return_value = succeed("launch")
def setUp(self):
"""
Fake supervisor, group and state
"""
self.tid = 'trans_id'
self.log = mock_log()
self.state = GroupState('tid', 'gid', 'g', {'s0': {'id': 's0'}}, {},
None, None, None, ScalingGroupStatus.ACTIVE,
desired=1)
self.group = mock_group(self.state)
self.gen_jobid = patch(self, 'otter.supervisor.generate_job_id',
return_value='jid')
self.supervisor = FakeSupervisor()
set_supervisor(self.supervisor)
self.addCleanup(set_supervisor, None)
self.group.view_config.return_value = succeed({'minEntities': 0})
self.group.view_launch_config.return_value = succeed('launch')
def test_authenticator(self, mock_ss, mock_ga, mock_reactor):
"""
Authenticator is generated and passed to SupervisorService
"""
self.addCleanup(lambda: set_supervisor(None))
makeService(test_config)
mock_ga.assert_called_once_with(mock_reactor, test_config['identity'])
self.assertIdentical(get_supervisor().authenticator,
mock_ga.return_value)
def test_authenticator(self, mock_ss, mock_ga, mock_reactor):
"""
Authenticator is generated and passed to SupervisorService
"""
self.addCleanup(lambda: set_supervisor(None))
makeService(test_config)
mock_ga.assert_called_once_with(mock_reactor, test_config['identity'])
self.assertIdentical(get_supervisor().authenticator,
mock_ga.return_value)
def test_supervisor_service_set_by_default(self, supervisor):
"""
A SupervisorService service is added to the Multiservice, and set as
default supervisor
"""
self.addCleanup(lambda: set_supervisor(None))
parent = makeService(test_config)
supervisor_service = parent.getServiceNamed('supervisor')
self.assertEqual(get_supervisor(), supervisor_service)
def test_supervisor_service_set_by_default(self, supervisor):
"""
A SupervisorService service is added to the Multiservice, and set as
default supervisor
"""
self.addCleanup(lambda: set_supervisor(None))
parent = makeService(test_config)
supervisor_service = parent.getServiceNamed('supervisor')
self.assertEqual(get_supervisor(), supervisor_service)
def test_health_checker_no_zookeeper(self, supervisor):
"""
A health checker is constructed by default with the store and kazoo health check
"""
self.addCleanup(lambda: set_supervisor(None))
self.assertIsNone(self.health_checker)
makeService(test_config)
self.assertIsNotNone(self.health_checker)
self.assertEqual(self.health_checker.checks['store'],
self.store.health_check)
self.assertEqual(self.health_checker.checks['kazoo'],
self.store.kazoo_health_check)
self.assertEqual(self.health_checker.checks['supervisor'],
get_supervisor().health_check)
def test_perform_eviction(self):
"""
Call supervisor's scrub metadata function.
"""
supervisor = FakeSupervisor()
set_supervisor(supervisor)
self.addCleanup(set_supervisor, None)
log, group = (object(), mock_group(None))
intent = EvictServerFromScalingGroup(log=log,
transaction_id='transaction_id',
scaling_group=group,
server_id='server_id')
r = sync_perform(
TypeDispatcher({
EvictServerFromScalingGroup:
partial(perform_evict_server, supervisor)
}), Effect(intent))
self.assertIsNone(r)
self.assertEqual(
supervisor.scrub_calls,
[(log, "transaction_id", group.tenant_id, 'server_id')])
def test_perform_eviction(self):
"""
Call supervisor's scrub metadata function.
"""
supervisor = FakeSupervisor()
set_supervisor(supervisor)
self.addCleanup(set_supervisor, None)
log, group = (object(), mock_group(None))
intent = EvictServerFromScalingGroup(
log=log, transaction_id='transaction_id',
scaling_group=group, server_id='server_id')
r = sync_perform(
TypeDispatcher({
EvictServerFromScalingGroup: partial(
perform_evict_server, supervisor)
}),
Effect(intent))
self.assertIsNone(r)
self.assertEqual(
supervisor.scrub_calls,
[(log, "transaction_id", group.tenant_id, 'server_id')])
def test_health_checker_no_zookeeper(self, supervisor):
"""
A health checker is constructed by default with the store and kazoo
health check
"""
self.addCleanup(lambda: set_supervisor(None))
self.assertIsNone(self.health_checker)
makeService(test_config)
self.assertIsNotNone(self.health_checker)
self.assertEqual(self.health_checker.checks['store'],
self.store.health_check)
self.assertEqual(self.health_checker.checks['kazoo'],
self.store.kazoo_health_check)
self.assertEqual(self.health_checker.checks['supervisor'],
get_supervisor().health_check)
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
s = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace')),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
get_consistency = partial(
get_consistency_level,
default=config_value('cassandra.default_consistency'),
exceptions=config_value('cassandra.consistency_exceptions'))
store = CassScalingGroupCollection(cassandra_cluster, reactor,
get_consistency)
admin_store = CassAdmin(cassandra_cluster, get_consistency)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
RetryingAuthenticator(
reactor,
ImpersonatingAuthenticator(
config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
max_retries=config_value('identity.max_retries'),
retry_interval=config_value('identity.retry_interval')),
cache_ttl)
supervisor = SupervisorService(authenticator.authenticate_tenant,
region, coiterate)
supervisor.setServiceParent(s)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
s.addService(FunctionalService(stop=partial(call_after_supervisor,
cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check,
es_host=config_value('elasticsearch.host'))
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(s)
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
kz_client = TxKazooClient(hosts=config_value('zookeeper.hosts'),
threads=threads, txlog=log.bind(system='kazoo'))
d = kz_client.start()
def on_client_ready(_):
# Setup scheduler service after starting
scheduler = setup_scheduler(s, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Set the client after starting
# NOTE: There is small amount of time when the start is not finished
# and the kz_client is not set in which case policy execution and group
# delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
s.addService(FunctionalService(stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return s
def tearDown(self):
"""
Reset the supervisor
"""
set_supervisor(None)
def tearDown(self):
"""
Reset the supervisor
"""
set_supervisor(None)
def tearDown(self):
"""
reset supervisor
"""
set_supervisor(None)
def tearDown(self):
"""
Revert mock Bobby client
"""
set_bobby(None)
set_supervisor(None)
def tearDown(self):
"""
Reset the supervisor
"""
set_supervisor(None)
set_config_data({})
def makeService(config):
"""
Set up the otter-api service.
"""
config = dict(config)
set_config_data(config)
parent = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'),
disconnect_on_cancel=True),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
store = CassScalingGroupCollection(
cassandra_cluster, reactor, config_value('limits.absolute.maxGroups'))
admin_store = CassAdmin(cassandra_cluster)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
service_configs = get_service_configs(config)
authenticator = generate_authenticator(reactor, config['identity'])
supervisor = SupervisorService(authenticator, region, coiterate,
service_configs)
supervisor.setServiceParent(parent)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
parent.addService(FunctionalService(stop=partial(
call_after_supervisor, cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check)
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(parent)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(parent)
# setup cloud feed
cf_conf = config.get('cloudfeeds', None)
if cf_conf is not None:
id_conf = deepcopy(config['identity'])
id_conf['strategy'] = 'single_tenant'
add_to_fanout(CloudFeedsObserver(
reactor=reactor,
authenticator=generate_authenticator(reactor, id_conf),
tenant_id=cf_conf['tenant_id'],
region=region,
service_configs=service_configs))
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
disable_logs = config_value('zookeeper.no_logs')
threadpool = ThreadPool(maxthreads=threads)
sync_kz_client = KazooClient(
hosts=config_value('zookeeper.hosts'),
# Keep trying to connect until the end of time with
# max interval of 10 minutes
connection_retry=dict(max_tries=-1, max_delay=600),
logger=None if disable_logs else TxLogger(log.bind(system='kazoo'))
)
kz_client = TxKazooClient(reactor, threadpool, sync_kz_client)
# Don't timeout. Keep trying to connect forever
d = kz_client.start(timeout=None)
def on_client_ready(_):
dispatcher = get_full_dispatcher(reactor, authenticator, log,
get_service_configs(config),
kz_client, store, supervisor,
cassandra_cluster)
# Setup scheduler service after starting
scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Give dispatcher to Otter REST object
otter.dispatcher = dispatcher
# Set the client after starting
# NOTE: There is small amount of time when the start is
# not finished and the kz_client is not set in which case
# policy execution and group delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
parent.addService(FunctionalService(
stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
setup_converger(
parent, kz_client, dispatcher,
config_value('converger.interval') or 10,
config_value('converger.build_timeout') or 3600,
config_value('converger.limited_retry_iterations') or 10,
config_value('converger.step_limits') or {})
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return parent
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
# Try to configure graylog and airbrake.
if config_value('graylog'):
if GraylogUDPPublisher is not None:
log.addObserver(
make_observer_chain(
GraylogUDPPublisher(**config_value('graylog')), False))
else:
warnings.warn("There is a configuration option for Graylog, but "
"txgraylog is not installed.")
if config_value('airbrake'):
if AirbrakeLogObserver is not None:
airbrake = AirbrakeLogObserver(
config_value('airbrake.api_key'),
config_value('environment'),
use_ssl=True
)
airbrake.start()
else:
warnings.warn("There is a configuration option for Airbrake, but "
"txairbrake is not installed.")
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'))
set_store(CassScalingGroupCollection(cassandra_cluster))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(
config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler'):
scheduler_service = SchedulerService(int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')),
cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
