def test_composition_single_tenant(self):
"""
authenticator is composed correctly with values from config
"""
r = mock.Mock()
self.config['strategy'] = 'single_tenant'
a = generate_authenticator(r, self.config)
self.assertIsInstance(a, CachingAuthenticator)
self.assertIdentical(a._reactor, r)
self.assertEqual(a._ttl, 50)
wa = a._authenticator
self.assertIsInstance(wa, WaitingAuthenticator)
self.assertIdentical(wa._reactor, r)
self.assertEqual(wa._wait, 4)
ra = wa._authenticator
self.assertIsInstance(ra, RetryingAuthenticator)
self.assertIdentical(ra._reactor, r)
self.assertEqual(ra._max_retries, 3)
self.assertEqual(ra._retry_interval, 5)
st = ra._authenticator
self.assertIsInstance(st, SingleTenantAuthenticator)
self.assertEqual(st._identity_user, 'uname')
self.assertEqual(st._identity_password, 'pwd')
self.assertEqual(st._url, 'htp')
def main(reactor):
parser = ArgumentParser(
description="Trigger convergence on all/some groups")
parser.add_argument(
"-c", dest="config", required=True,
help="Config file containing identity and cassandra info")
parser.add_argument(
"-g", nargs="+", dest="group",
help=("Group(s) to trigger. Should be in tenantId:groupId form. "
"If not provided convergence will be triggerred on all groups "
"in CASS"))
parser.add_argument("-l", dest="limit", type=int, default=10,
help="Concurrency limit. Defaults to 10")
parsed = parser.parse_args()
conf = json.load(open(parsed.config))
cass_client = connect_cass_servers(reactor, conf["cassandra"])
authenticator = generate_authenticator(reactor, conf["identity"])
store = CassScalingGroupCollection(cass_client, reactor, 1000)
if parsed.group:
groups = [g.split(":") for g in parsed.group]
groups = [{"tenantId": tid, "groupId": gid} for tid, gid in groups]
else:
groups = yield store.get_all_groups()
yield trigger_convergence_groups(authenticator, conf["region"], groups,
parsed.limit)
yield cass_client.disconnect()
def test_composition_impersonation(self):
"""
authenticator is composed correctly with values from config
"""
r = mock.Mock()
a = generate_authenticator(r, self.config)
self.assertIsInstance(a, CachingAuthenticator)
self.assertIdentical(a._reactor, r)
self.assertEqual(a._ttl, 50)
wa = a._authenticator
self.assertIsInstance(wa, WaitingAuthenticator)
self.assertIdentical(wa._reactor, r)
self.assertEqual(wa._wait, 4)
ra = wa._authenticator
self.assertIsInstance(ra, RetryingAuthenticator)
self.assertIdentical(ra._reactor, r)
self.assertEqual(ra._max_retries, 3)
self.assertEqual(ra._retry_interval, 5)
ia = ra._authenticator
self.assertIsInstance(ia, ImpersonatingAuthenticator)
self.assertEqual(ia._identity_admin_user, 'uname')
self.assertEqual(ia._identity_admin_password, 'pwd')
self.assertEqual(ia._url, 'htp')
self.assertEqual(ia._admin_url, 'ad')
def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def test_composition_single_tenant(self):
"""
authenticator is composed correctly with values from config
"""
r = mock.Mock()
self.config['strategy'] = 'single_tenant'
a = generate_authenticator(r, self.config)
self.assertIsInstance(a, CachingAuthenticator)
self.assertIdentical(a._reactor, r)
self.assertEqual(a._ttl, 50)
wa = a._authenticator
self.assertIsInstance(wa, WaitingAuthenticator)
self.assertIdentical(wa._reactor, r)
self.assertEqual(wa._wait, 4)
ra = wa._authenticator
self.assertIsInstance(ra, RetryingAuthenticator)
self.assertIdentical(ra._reactor, r)
self.assertEqual(ra._max_retries, 3)
self.assertEqual(ra._retry_interval, 5)
st = ra._authenticator
self.assertIsInstance(st, SingleTenantAuthenticator)
self.assertEqual(st._identity_user, 'uname')
self.assertEqual(st._identity_password, 'pwd')
self.assertEqual(st._url, 'htp')
def main(reactor):
parser = ArgumentParser(description="Trigger convergence on all/some groups")
parser.add_argument("-c", dest="config", required=True, help="Config file containing identity and cassandra info")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("-g", nargs="+", dest="group", help="Group(s) to trigger. Should be in tenantId:groupId form")
group.add_argument("-t", nargs="+", dest="tenant_id", help="TenantID(s) whose group's to trigger")
group.add_argument(
"--conf-conv-tenants",
action="store_true",
help=("Convergence triggered on tenants configured as " '"convergence-tenants" setting config file'),
)
group.add_argument(
"--conf-non-conv-tenants",
action="store_true",
dest="disabled_tenants",
help=("Convergence triggered on all tenants except ones in " '"non-convergence-tenants" setting in conf file'),
)
group.add_argument("--all", action="store_true", help="Convergence will be triggered on all groups")
parser.add_argument("-l", dest="limit", type=int, default=10, help="Concurrency limit. Defaults to 10")
parser.add_argument("--no-error-group", action="store_true", help="Do not converge ERROR groups")
parsed = parser.parse_args()
conf = json.load(open(parsed.config))
cass_client = connect_cass_servers(reactor, conf["cassandra"])
authenticator = generate_authenticator(reactor, conf["identity"])
store = CassScalingGroupCollection(cass_client, reactor, 1000)
groups = yield get_groups(parsed, store, conf)
yield trigger_convergence_groups(authenticator, conf["region"], groups, parsed.limit, parsed.no_error_group)
yield cass_client.disconnect()
def test_composition_impersonation(self):
"""
authenticator is composed correctly with values from config
"""
r = mock.Mock()
a = generate_authenticator(r, self.config)
self.assertIsInstance(a, CachingAuthenticator)
self.assertIdentical(a._reactor, r)
self.assertEqual(a._ttl, 50)
wa = a._authenticator
self.assertIsInstance(wa, WaitingAuthenticator)
self.assertIdentical(wa._reactor, r)
self.assertEqual(wa._wait, 4)
ra = wa._authenticator
self.assertIsInstance(ra, RetryingAuthenticator)
self.assertIdentical(ra._reactor, r)
self.assertEqual(ra._max_retries, 3)
self.assertEqual(ra._retry_interval, 5)
ia = ra._authenticator
self.assertIsInstance(ia, ImpersonatingAuthenticator)
self.assertEqual(ia._identity_admin_user, 'uname')
self.assertEqual(ia._identity_admin_password, 'pwd')
self.assertEqual(ia._url, 'htp')
self.assertEqual(ia._admin_url, 'ad')
def __init__(self, reactor, config, log, clock=None, collect=None):
"""
Initialize the service by connecting to Cassandra and setting up
authenticator
:param reactor: Twisted reactor for connection purposes
:param dict config: All the config necessary to run the service.
Comes from config file
:param IReactorTime clock: Optional reactor for timer purpose
"""
self._client = connect_cass_servers(reactor, config['cassandra'])
self.log = log
self.reactor = reactor
self._divergent_groups = {}
self.divergent_timeout = get_in(
['metrics', 'divergent_timeout'], config, 3600)
self._service = TimerService(
get_in(['metrics', 'interval'], config, default=60),
collect or self.collect,
reactor,
config,
self.log,
client=self._client,
authenticator=generate_authenticator(reactor, config['identity']))
self._service.clock = clock or reactor
def test_cache_ttl_defaults(self):
"""
CachingAuthenticator is created with default of 300 if not given
"""
del self.config['cache_ttl']
r = mock.Mock()
a = generate_authenticator(r, self.config)
self.assertEqual(a._ttl, 300)
def test_wait_defaults(self):
"""
WaitingAuthenticator is created with default of 5 if not given
"""
del self.config['wait']
r = mock.Mock()
a = generate_authenticator(r, self.config)
self.assertEqual(a._authenticator._wait, 5)
def test_cache_ttl_defaults(self):
"""
CachingAuthenticator is created with default of 300 if not given
"""
del self.config['cache_ttl']
r = mock.Mock()
a = generate_authenticator(r, self.config)
self.assertEqual(a._ttl, 300)
def test_wait_defaults(self):
"""
WaitingAuthenticator is created with default of 5 if not given
"""
del self.config['wait']
r = mock.Mock()
a = generate_authenticator(r, self.config)
self.assertEqual(a._authenticator._wait, 5)
def collect_metrics(reactor, config, log, client=None, authenticator=None,
_print=False):
"""
Start collecting the metrics
:param reactor: Twisted reactor
:param dict config: Configuration got from file containing all info
needed to collect metrics
:param :class:`silverberg.client.CQLClient` client:
Optional cassandra client. A new client will be created
if this is not given and disconnected before returing
:param :class:`otter.auth.IAuthenticator` authenticator:
Optional authenticator. A new authenticator will be created
if this is not given
:param bool _print: Should debug messages be printed to stdout?
:return: :class:`Deferred` fired with ``list`` of `GroupMetrics`
"""
_client = client or connect_cass_servers(reactor, config['cassandra'])
authenticator = authenticator or generate_authenticator(reactor,
config['identity'])
store = CassScalingGroupCollection(_client, reactor, 1000)
dispatcher = get_dispatcher(reactor, authenticator, log,
get_service_configs(config), store)
# calculate metrics on launch_server and non-paused groups
groups = yield perform(dispatcher, Effect(GetAllValidGroups()))
groups = [
g for g in groups
if json.loads(g["launch_config"]).get("type") == "launch_server" and
(not g.get("paused", False))]
tenanted_groups = groupby(lambda g: g["tenantId"], groups)
group_metrics = yield get_all_metrics(
dispatcher, tenanted_groups, log, _print=_print)
# Add to cloud metrics
metr_conf = config.get("metrics", None)
if metr_conf is not None:
eff = add_to_cloud_metrics(
metr_conf['ttl'], config['region'], group_metrics,
len(tenanted_groups), config, log, _print)
eff = Effect(TenantScope(eff, metr_conf['tenant_id']))
yield perform(dispatcher, eff)
log.msg('added to cloud metrics')
if _print:
print('added to cloud metrics')
if _print:
group_metrics.sort(key=lambda g: abs(g.desired - g.actual),
reverse=True)
print('groups sorted as per divergence')
print('\n'.join(map(str, group_metrics)))
# Disconnect only if we created the client
if not client:
yield _client.disconnect()
defer.returnValue(group_metrics)
def collect_metrics(reactor, config, log, client=None, authenticator=None,
_print=False):
"""
Start collecting the metrics
:param reactor: Twisted reactor
:param dict config: Configuration got from file containing all info
needed to collect metrics
:param :class:`silverberg.client.CQLClient` client:
Optional cassandra client. A new client will be created
if this is not given and disconnected before returing
:param :class:`otter.auth.IAuthenticator` authenticator:
Optional authenticator. A new authenticator will be created
if this is not given
:param bool _print: Should debug messages be printed to stdout?
:return: :class:`Deferred` fired with ``list`` of `GroupMetrics`
"""
convergence_tids = config.get('convergence-tenants', [])
_client = client or connect_cass_servers(reactor, config['cassandra'])
authenticator = authenticator or generate_authenticator(reactor,
config['identity'])
store = CassScalingGroupCollection(_client, reactor, 1000)
dispatcher = get_dispatcher(reactor, authenticator, log,
get_service_configs(config), store)
# calculate metrics
fpath = get_in(["metrics", "last_tenant_fpath"], config,
default="last_tenant.txt")
tenanted_groups = yield perform(
dispatcher,
get_todays_scaling_groups(convergence_tids, fpath))
group_metrics = yield get_all_metrics(
dispatcher, tenanted_groups, log, _print=_print)
# Add to cloud metrics
metr_conf = config.get("metrics", None)
if metr_conf is not None:
eff = add_to_cloud_metrics(
metr_conf['ttl'], config['region'], group_metrics,
len(tenanted_groups), log, _print)
eff = Effect(TenantScope(eff, metr_conf['tenant_id']))
yield perform(dispatcher, eff)
log.msg('added to cloud metrics')
if _print:
print('added to cloud metrics')
if _print:
group_metrics.sort(key=lambda g: abs(g.desired - g.actual),
reverse=True)
print('groups sorted as per divergence', *group_metrics, sep='\n')
# Disconnect only if we created the client
if not client:
yield _client.disconnect()
defer.returnValue(group_metrics)
def main(reactor):
conf = json.load(open(sys.argv[1]))
conf = conf["default_attributes"]["otter"]["config"]
conf["identity"]["strategy"] = "single_tenant"
conf["identity"]["username"] = os.environ["OTTERPROD_UNAME"]
conf["identity"]["password"] = os.environ["OTTERPROD_PWD"]
authenticator = generate_authenticator(reactor, conf["identity"])
token, catalog = yield authenticator.authenticate_tenant("764771")
all_tenants = set((yield get_tenant_ids(token, catalog)))
worker_tenants = all_tenants - set(conf["convergence-tenants"])
print(*worker_tenants, sep='\n')
def main(reactor):
conf = json.load(open(sys.argv[1]))
conf = conf["default_attributes"]["otter"]["config"]
conf["identity"]["strategy"] = "single_tenant"
conf["identity"]["username"] = os.environ["OTTERPROD_UNAME"]
conf["identity"]["password"] = os.environ["OTTERPROD_PWD"]
authenticator = generate_authenticator(reactor, conf["identity"])
token, catalog = yield authenticator.authenticate_tenant("764771")
all_tenants = set((yield get_tenant_ids(token, catalog)))
worker_tenants = all_tenants - set(conf["convergence-tenants"])
print(*worker_tenants, sep='\n')
def main(reactor):
parser = ArgumentParser(
description="Trigger convergence on all/some groups")
parser.add_argument(
"-c", dest="config", required=True,
help="Config file containing identity and cassandra info")
parser.add_argument(
"--steps", action="store_true",
help=("Return steps that would be taken if convergence was triggered "
"with desired set to current actual. No convergence triggered"))
parser.add_argument(
"--set-desired-to-actual", action="store_true", dest="set_desired",
help="Set group's desired to current actual number of servers")
parser.add_argument(
"--pause", action="store_true", dest="pause_group",
help="Pause given groups")
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
"-g", nargs="+", dest="group",
help="Group(s) to trigger. Should be in tenantId:groupId form")
group.add_argument(
"-t", nargs="+", dest="tenant_id",
help="TenantID(s) whose group's to trigger")
group.add_argument(
"--conf-conv-tenants", action="store_true",
help=("Convergence triggered on tenants configured as "
"\"convergence-tenants\" setting config file"))
group.add_argument(
"--conf-non-conv-tenants", action="store_true",
dest="disabled_tenants",
help=("Convergence triggered on all tenants except ones in "
"\"non-convergence-tenants\" setting in conf file"))
group.add_argument("--all", action="store_true",
help="Convergence will be triggered on all groups")
parser.add_argument("-l", dest="limit", type=int, default=10,
help="Concurrency limit. Defaults to 10")
parsed = parser.parse_args()
conf = json.load(open(parsed.config))
set_config_data(conf)
cass_client = connect_cass_servers(reactor, conf["cassandra"])
authenticator = generate_authenticator(reactor, conf["identity"])
store = CassScalingGroupCollection(cass_client, reactor, 1000)
groups = yield get_groups(parsed, store, conf)
if parsed.steps:
steps = yield groups_steps(groups, reactor, store, cass_client,
authenticator, conf)
pprint(steps)
elif parsed.set_desired:
yield set_desired_to_actual(groups, reactor, store, cass_client,
authenticator, conf)
elif parsed.pause_group:
yield pause_groups(cass_client, groups)
else:
error_groups = yield trigger_convergence_groups(
authenticator, conf["region"], groups, parsed.limit)
if error_groups:
print("Following groups errored")
pprint(error_groups)
yield cass_client.disconnect()
def makeService(config):
"""
Set up the otter-api service.
"""
config = dict(config)
set_config_data(config)
parent = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'),
disconnect_on_cancel=True),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
store = CassScalingGroupCollection(
cassandra_cluster, reactor, config_value('limits.absolute.maxGroups'))
admin_store = CassAdmin(cassandra_cluster)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
service_configs = get_service_configs(config)
authenticator = generate_authenticator(reactor, config['identity'])
supervisor = SupervisorService(authenticator, region, coiterate,
service_configs)
supervisor.setServiceParent(parent)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
parent.addService(FunctionalService(stop=partial(
call_after_supervisor, cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check)
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(parent)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(parent)
# setup cloud feed
cf_conf = config.get('cloudfeeds', None)
if cf_conf is not None:
id_conf = deepcopy(config['identity'])
id_conf['strategy'] = 'single_tenant'
add_to_fanout(CloudFeedsObserver(
reactor=reactor,
authenticator=generate_authenticator(reactor, id_conf),
tenant_id=cf_conf['tenant_id'],
region=region,
service_configs=service_configs))
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
disable_logs = config_value('zookeeper.no_logs')
threadpool = ThreadPool(maxthreads=threads)
sync_kz_client = KazooClient(
hosts=config_value('zookeeper.hosts'),
# Keep trying to connect until the end of time with
# max interval of 10 minutes
connection_retry=dict(max_tries=-1, max_delay=600),
logger=None if disable_logs else TxLogger(log.bind(system='kazoo'))
)
kz_client = TxKazooClient(reactor, threadpool, sync_kz_client)
# Don't timeout. Keep trying to connect forever
d = kz_client.start(timeout=None)
def on_client_ready(_):
dispatcher = get_full_dispatcher(reactor, authenticator, log,
get_service_configs(config),
kz_client, store, supervisor,
cassandra_cluster)
# Setup scheduler service after starting
scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Give dispatcher to Otter REST object
otter.dispatcher = dispatcher
# Set the client after starting
# NOTE: There is small amount of time when the start is
# not finished and the kz_client is not set in which case
# policy execution and group delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
parent.addService(FunctionalService(
stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
setup_converger(
parent, kz_client, dispatcher,
config_value('converger.interval') or 10,
config_value('converger.build_timeout') or 3600,
config_value('converger.limited_retry_iterations') or 10,
config_value('converger.step_limits') or {})
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return parent
def main(reactor):
parser = ArgumentParser(
description="Trigger convergence on all/some groups")
parser.add_argument(
"-c",
dest="config",
required=True,
help="Config file containing identity and cassandra info")
parser.add_argument(
"--steps",
action="store_true",
help=("Return steps that would be taken if convergence was triggered "
"with desired set to current actual. No convergence triggered"))
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
"-g",
nargs="+",
dest="group",
help="Group(s) to trigger. Should be in tenantId:groupId form")
group.add_argument("-t",
nargs="+",
dest="tenant_id",
help="TenantID(s) whose group's to trigger")
group.add_argument("--conf-conv-tenants",
action="store_true",
help=("Convergence triggered on tenants configured as "
"\"convergence-tenants\" setting config file"))
group.add_argument(
"--conf-non-conv-tenants",
action="store_true",
dest="disabled_tenants",
help=("Convergence triggered on all tenants except ones in "
"\"non-convergence-tenants\" setting in conf file"))
group.add_argument("--all",
action="store_true",
help="Convergence will be triggered on all groups")
parser.add_argument("-l",
dest="limit",
type=int,
default=10,
help="Concurrency limit. Defaults to 10")
parser.add_argument("--no-error-group",
action="store_true",
help="Do not converge ERROR groups")
parsed = parser.parse_args()
conf = json.load(open(parsed.config))
cass_client = connect_cass_servers(reactor, conf["cassandra"])
authenticator = generate_authenticator(reactor, conf["identity"])
store = CassScalingGroupCollection(cass_client, reactor, 1000)
groups = yield get_groups(parsed, store, conf)
if parsed.steps:
steps = yield groups_steps(groups, reactor, store, cass_client,
authenticator, conf)
print(*steps, sep='\n')
else:
yield trigger_convergence_groups(authenticator, conf["region"], groups,
parsed.limit, parsed.no_error_group)
yield cass_client.disconnect()
