def on_client_ready(_):
dispatcher = get_full_dispatcher(reactor, authenticator, log,
get_service_configs(config),
kz_client, store, supervisor,
cassandra_cluster)
# Setup scheduler service after starting
scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Give dispatcher to Otter REST object
otter.dispatcher = dispatcher
# Set the client after starting
# NOTE: There is small amount of time when the start is
# not finished and the kz_client is not set in which case
# policy execution and group delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
parent.addService(FunctionalService(
stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
setup_converger(
parent, kz_client, dispatcher,
config_value('converger.interval') or 10,
config_value('converger.build_timeout') or 3600,
config_value('converger.limited_retry_iterations') or 10,
config_value('converger.step_limits') or {})
def delete_server(log, region, service_catalog, auth_token, instance_details):
"""
Delete the server specified by instance_details.
TODO: Load balancer draining.
:param str region: A rackspace region as found in the service catalog.
:param list service_catalog: A list of services as returned by the auth apis.
:param str auth_token: The user's auth token.
:param tuple instance_details: A 2-tuple of server_id and a list of
load balancer Add Node responses.
Example::
('da08965f-4c2d-41aa-b492-a3c02706202f',
[('12345',
{'nodes': [{'id': 'a', 'address': ... }]}),
('54321',
{'nodes': [{'id': 'b', 'address': ... }]})])
:return: TODO
"""
lb_region = config_value('regionOverrides.cloudLoadBalancers') or region
cloudLoadBalancers = config_value('cloudLoadBalancers')
cloudServersOpenStack = config_value('cloudServersOpenStack')
log.msg("Looking for load balancer endpoint: %(service_name)s",
service_name=cloudLoadBalancers,
region=lb_region)
lb_endpoint = public_endpoint_url(service_catalog, cloudLoadBalancers,
lb_region)
log.msg("Looking for cloud servers endpoint: %(service_name)s",
service_name=cloudServersOpenStack,
region=region)
server_endpoint = public_endpoint_url(service_catalog,
cloudServersOpenStack, region)
(server_id, loadbalancer_details) = instance_details
node_info = itertools.chain(*[[(loadbalancer_id, node['id'])
for node in node_details['nodes']]
for (loadbalancer_id,
node_details) in loadbalancer_details])
d = gatherResults([
remove_from_load_balancer(lb_endpoint, auth_token, loadbalancer_id,
node_id)
for (loadbalancer_id, node_id) in node_info
],
consumeErrors=True)
def when_removed_from_loadbalancers(_ignore):
return verified_delete(log, server_endpoint, auth_token, server_id)
d.addCallback(when_removed_from_loadbalancers)
return d
def test_update_config_existing(self):
"""
:func:`~config.update_config_data` will update existing config
and not remove others
"""
config.update_config_data("baz.bax", "new")
self.assertEqual(config.config_value("baz.bax"), "new")
self.assertEqual(config.config_value("foo"), "bar")
def delete_server(log, region, service_catalog, auth_token, instance_details):
"""
Delete the server specified by instance_details.
TODO: Load balancer draining.
:param str region: A rackspace region as found in the service catalog.
:param list service_catalog: A list of services as returned by the auth apis.
:param str auth_token: The user's auth token.
:param tuple instance_details: A 2-tuple of server_id and a list of
load balancer Add Node responses.
Example::
('da08965f-4c2d-41aa-b492-a3c02706202f',
[('12345',
{'nodes': [{'id': 'a', 'address': ... }]}),
('54321',
{'nodes': [{'id': 'b', 'address': ... }]})])
:return: TODO
"""
lb_region = config_value('regionOverrides.cloudLoadBalancers') or region
cloudLoadBalancers = config_value('cloudLoadBalancers')
cloudServersOpenStack = config_value('cloudServersOpenStack')
log.msg("Looking for load balancer endpoint: %(service_name)s",
service_name=cloudLoadBalancers,
region=lb_region)
lb_endpoint = public_endpoint_url(service_catalog,
cloudLoadBalancers,
lb_region)
log.msg("Looking for cloud servers endpoint: %(service_name)s",
service_name=cloudServersOpenStack,
region=region)
server_endpoint = public_endpoint_url(service_catalog,
cloudServersOpenStack,
region)
(server_id, loadbalancer_details) = instance_details
node_info = itertools.chain(
*[[(loadbalancer_id, node['id']) for node in node_details['nodes']]
for (loadbalancer_id, node_details) in loadbalancer_details])
d = gatherResults(
[remove_from_load_balancer(lb_endpoint, auth_token, loadbalancer_id, node_id)
for (loadbalancer_id, node_id) in node_info], consumeErrors=True)
def when_removed_from_loadbalancers(_ignore):
return verified_delete(log, server_endpoint, auth_token, server_id)
d.addCallback(when_removed_from_loadbalancers)
return d
def remove_from_load_balancer(log, endpoint, auth_token, loadbalancer_id,
node_id, clock=None):
"""
Remove a node from a load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str loadbalancer_id: The ID for a cloud loadbalancer.
:param str node_id: The ID for a node in that cloudloadbalancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id), 'nodes', str(node_id))
def check_422_deleted(failure):
# A LB being deleted sometimes results in a 422. This function
# unfortunately has to parse the body of the message to see if this is an
# acceptable 422 (if the LB has been deleted or the node has already been
# removed, then 'removing from load balancer' as a task should be
# successful - if the LB is in ERROR, then nothing more can be done to
# it except resetting it - may as well remove the server.)
failure.trap(APIError)
error = failure.value
if error.code == 422:
message = json.loads(error.body)['message']
if ('load balancer is deleted' not in message and
'PENDING_DELETE' not in message):
return failure
lb_log.msg(message)
else:
return failure
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
# Success is 200/202. An LB not being found is 404. A node not being
# found is a 404. But a deleted LB sometimes results in a 422.
d.addCallback(log_on_response_code, lb_log, 'Node to delete does not exist', 404)
d.addCallback(check_success, [200, 202, 404])
d.addCallback(treq.content) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(check_422_deleted)
d.addErrback(log_lb_unexpected_errors, path, lb_log, 'remove_node')
return d
d = retry(
remove,
can_retry=retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def test_update_config_new(self):
"""
:func:`~config.update_config_data` will add new config and not remove
others
"""
config.update_config_data("baz.new", "wha")
config.update_config_data("baz.some.other", "who")
self.assertEqual(config.config_value("baz.new"), "wha")
self.assertEqual(config.config_value("baz.some.other"), "who")
self.assertEqual(config.config_value("baz.bax"), "quux")
def add_to_load_balancer(log, endpoint, auth_token, lb_config, ip_address, undo, clock=None):
"""
Add an IP addressed to a load balancer based on the lb_config.
TODO: Handle load balancer node metadata.
:param log: A bound logger
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str lb_config: An lb_config dictionary.
:param str ip_address: The IP Address of the node to add to the load
balancer.
:param IUndoStack undo: An IUndoStack to push any reversable operations onto.
:return: Deferred that fires with the Add Node to load balancer response
as a dict.
"""
lb_id = lb_config['loadBalancerId']
port = lb_config['port']
path = append_segments(endpoint, 'loadbalancers', str(lb_id), 'nodes')
lb_log = log.bind(loadbalancer_id=lb_id, ip_address=ip_address)
def add():
d = treq.post(path, headers=headers(auth_token),
data=json.dumps({"nodes": [{"address": ip_address,
"port": port,
"condition": "ENABLED",
"type": "PRIMARY"}]}),
log=lb_log)
d.addCallback(check_success, [200, 202])
d.addErrback(log_lb_unexpected_errors, lb_log, 'add_node')
d.addErrback(wrap_request_error, path, 'add_node')
d.addErrback(check_deleted_clb, lb_id)
return d
d = retry(
add,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
def when_done(result):
lb_log.msg('Added to load balancer', node_id=result['nodes'][0]['id'])
undo.push(remove_from_load_balancer,
lb_log,
endpoint,
auth_token,
lb_id,
result['nodes'][0]['id'])
return result
return d.addCallback(treq.json_content).addCallback(when_done)
def generate_authenticator(reactor):
"""
Generate authenticator
"""
# REVIEW: Seperating out to test. Have any better idea?
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
return CachingAuthenticator(
reactor,
WaitingAuthenticator(
reactor,
RetryingAuthenticator(
reactor,
ImpersonatingAuthenticator(
config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
max_retries=config_value('identity.max_retries'),
retry_interval=config_value('identity.retry_interval')),
config_value('identity.wait') or 5),
cache_ttl)
def _remove_from_clb(log,
endpoint,
auth_token,
loadbalancer_id,
node_id,
clock=None):
"""
Remove a node from a CLB load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone authentication token.
:param str loadbalancer_id: The ID for a Cloud Load Balancer.
:param str node_id: The ID for a node in that Cloud Load Balancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id),
'nodes', str(node_id))
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
d.addCallback(check_success, [200, 202])
d.addCallback(treq.content
) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(log_lb_unexpected_errors, lb_log, 'remove_node')
d.addErrback(wrap_request_error, path, 'remove_node')
d.addErrback(check_deleted_clb, loadbalancer_id, node_id)
return d
d = retry(remove,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(
config_value('worker.lb_max_retries')
or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range')
or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
# A node or CLB deleted is considered successful removal
d.addErrback(
lambda f: f.trap(CLBOrNodeDeleted) and lb_log.msg(f.value.message))
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def add_to_load_balancer(log,
request_bag,
lb_config,
server_details,
undo,
clock=None):
"""
Adds a given server to a given load balancer.
:param log: A bound logger.
:param callable request_bag: A request function.
:param str lb_config: An ``lb_config`` dictionary specifying which load
balancer to add the server to.
:param dict server_details: The server details, as returned by Nova.
:return: Deferred that fires with the load balancer response. The
structure of this object depends on the load balancer type.
"""
lb_type = lb_config.get("type", "CloudLoadBalancer")
if lb_type == "CloudLoadBalancer":
cloudLoadBalancers = config_value('cloudLoadBalancers')
endpoint = public_endpoint_url(request_bag.service_catalog,
cloudLoadBalancers,
request_bag.lb_region)
auth_token = request_bag.auth_token
ip_address = _servicenet_address(server_details["server"])
return add_to_clb(log, endpoint, auth_token, lb_config, ip_address,
undo, clock)
elif lb_type == "RackConnectV3":
lb_id = lb_config["loadBalancerId"]
server_id = server_details["server"]["id"]
return add_to_rcv3(request_bag, lb_id, server_id)
else:
raise RuntimeError(
"Unknown cloud load balancer type! config: {}".format(lb_config))
def scrub_otter_metadata(log,
auth_token,
service_catalog,
region,
server_id,
_treq=treq):
"""
Scrub otter-specific management metadata from the server.
:param BoundLog log: The bound logger instance.
:param str auth_token: Keystone auth token.
:param str region: The region the server is in.
:param str server_id: The id of the server to remove metadata from.
:param _treq: The treq instance; possibly a test double.
"""
bound_log = log.bind(region=region, server_id=server_id)
bound_log.msg("Scrubbing otter-specific metadata")
service_name = config_value('cloudServersOpenStack')
endpoint = public_endpoint_url(service_catalog, service_name, region)
url = append_segments(endpoint, 'servers', server_id, 'metadata')
auth_hdr = headers(auth_token)
get, put = [
lambda data=None, method=method: _treq.request(
method, url, headers=auth_hdr, data=data, log=bound_log)
for method in ["GET", "PUT"]
]
return (get().addCallback(_treq.json_content).addCallback(
comp(json.dumps,
_without_otter_metadata)).addCallback(put).addCallback(
_treq.content))
def _get_request_bag(self, log, scaling_group):
"""
Builds :obj:`RequestBag` containing a bunch of useful stuff for making
HTTP requests.
"""
tenant_id = scaling_group.tenant_id
dispatcher = get_legacy_dispatcher(reactor, self.authenticator, log,
self.service_configs)
lb_region = config_value('regionOverrides.cloudLoadBalancers')
def authenticate():
log.msg("Authenticating for tenant")
d = self.authenticator.authenticate_tenant(tenant_id, log=log)
def when_authenticated((auth_token, service_catalog)):
bag = RequestBag(
lb_region=lb_region or self.region,
region=self.region,
dispatcher=dispatcher,
tenant_id=tenant_id,
auth_token=auth_token,
service_catalog=service_catalog,
re_auth=authenticate,
)
return bag
return d.addCallback(when_authenticated)
return authenticate()
def when_authenticated((auth_token, service_catalog)):
return launch_server_v1.delete_server(
log,
config_value('region'),
service_catalog,
auth_token,
(server['id'], server['lb_info']))
def create_webhooks(self, policy_id, data):
"""
see :meth:`otter.models.interface.IScalingGroup.create_webhooks`
"""
if self.error is not None:
return defer.fail(self.error)
if policy_id in self.policies:
max_webhooks = config_value('limits.absolute.maxWebhooksPerPolicy')
curr_webhooks = len(self.webhooks.get(policy_id, []))
if len(data) + curr_webhooks > max_webhooks:
return defer.fail(
WebhooksOverLimitError(self.tenant_id, self.uuid,
policy_id, max_webhooks,
curr_webhooks, len(data)))
created = []
for webhook_input in data:
webhook_real = {'metadata': {}}
webhook_real.update(webhook_input)
webhook_real['capability'] = {}
(webhook_real['capability']['version'],
webhook_real['capability']['hash']) = generate_capability()
uuid = str(uuid4())
self.webhooks[policy_id][uuid] = webhook_real
# return a copy so this store doesn't get mutated
created.append(dict(id=uuid, **webhook_real))
return defer.succeed(created)
else:
return defer.fail(NoSuchPolicyError(self.tenant_id,
self.uuid, policy_id))
def add_to_load_balancer(log, request_bag, lb_config, server_details, undo,
clock=None):
"""
Adds a given server to a given load balancer.
:param log: A bound logger.
:param callable request_bag: A request function.
:param str lb_config: An ``lb_config`` dictionary specifying which load
balancer to add the server to.
:param dict server_details: The server details, as returned by Nova.
:return: Deferred that fires with the load balancer response. The
structure of this object depends on the load balancer type.
"""
lb_type = lb_config.get("type", "CloudLoadBalancer")
if lb_type == "CloudLoadBalancer":
cloudLoadBalancers = config_value('cloudLoadBalancers')
endpoint = public_endpoint_url(request_bag.service_catalog,
cloudLoadBalancers,
request_bag.lb_region)
auth_token = request_bag.auth_token
ip_address = _servicenet_address(server_details["server"])
return add_to_clb(log, endpoint, auth_token, lb_config, ip_address,
undo, clock)
elif lb_type == "RackConnectV3":
lb_id = lb_config["loadBalancerId"]
server_id = server_details["server"]["id"]
return add_to_rcv3(request_bag, lb_id, server_id)
else:
raise RuntimeError("Unknown cloud load balancer type! config: {}"
.format(lb_config))
def scrub_otter_metadata(log, auth_token, service_catalog, region, server_id,
_treq=treq):
"""
Scrub otter-specific management metadata from the server.
:param BoundLog log: The bound logger instance.
:param str auth_token: Keystone auth token.
:param str region: The region the server is in.
:param str server_id: The id of the server to remove metadata from.
:param _treq: The treq instance; possibly a test double.
"""
bound_log = log.bind(region=region, server_id=server_id)
bound_log.msg("Scrubbing otter-specific metadata")
service_name = config_value('cloudServersOpenStack')
endpoint = public_endpoint_url(service_catalog, service_name, region)
url = append_segments(endpoint, 'servers', server_id, 'metadata')
auth_hdr = headers(auth_token)
get, put = [lambda data=None, method=method: _treq.request(
method, url, headers=auth_hdr, data=data, log=bound_log)
for method in ["GET", "PUT"]]
return (get()
.addCallback(_treq.json_content)
.addCallback(comp(json.dumps, _without_otter_metadata))
.addCallback(put)
.addCallback(_treq.content))
def when_removed_from_loadbalancers(_ignore):
cloudServersOpenStack = config_value('cloudServersOpenStack')
server_endpoint = public_endpoint_url(request_bag.service_catalog,
cloudServersOpenStack,
request_bag.region)
return verified_delete(log, server_endpoint, request_bag, server_id,
clock=clock)
def _get_request_bag(self, log, scaling_group):
"""
Builds :obj:`RequestBag` containing a bunch of useful stuff for making
HTTP requests.
"""
tenant_id = scaling_group.tenant_id
dispatcher = get_legacy_dispatcher(reactor, self.authenticator, log,
self.service_configs)
lb_region = config_value('regionOverrides.cloudLoadBalancers')
def authenticate():
log.msg("Authenticating for tenant")
d = self.authenticator.authenticate_tenant(tenant_id, log=log)
def when_authenticated((auth_token, service_catalog)):
bag = RequestBag(
lb_region=lb_region or self.region,
region=self.region,
dispatcher=dispatcher,
tenant_id=tenant_id,
auth_token=auth_token,
service_catalog=service_catalog,
re_auth=authenticate,
)
return bag
return d.addCallback(when_authenticated)
return authenticate()
def remove_from_load_balancer(log, request_bag, lb_config, lb_response,
clock=None):
"""
Remove a node from a load balancer.
:param BoundLog log: A bound logger.
:param request_bag: A request function.
:param dict lb_config: An ``lb_config`` dictionary.
:param lb_response: The response the load balancer provided when the server
being removed was added. Type and shape is dependant on type of load
balancer.
:param IReactorTime clock: An optional clock, for testing. Will be passed
on to implementations of node removal logic for specific load balancer
APIs, if they support a clock.
:returns: A Deferred that fires with :data:`None` if the operation
completed successfully, or errbacks with an RequestError.
"""
lb_type = lb_config.get("type", "CloudLoadBalancer")
if lb_type == "CloudLoadBalancer":
cloudLoadBalancers = config_value('cloudLoadBalancers')
endpoint = public_endpoint_url(request_bag.service_catalog,
cloudLoadBalancers,
request_bag.lb_region)
auth_token = request_bag.auth_token
loadbalancer_id = lb_config["loadBalancerId"]
node_id = next(node_info["id"] for node_info in lb_response["nodes"])
return _remove_from_clb(log, endpoint, auth_token, loadbalancer_id,
node_id, clock)
elif lb_type == "RackConnectV3":
lb_id = lb_config["loadBalancerId"]
node_id = next(pair["cloud_server"]["id"] for pair in lb_response)
return remove_from_rcv3(request_bag, lb_id, node_id)
else:
raise RuntimeError("Unknown cloud load balancer type! config: {}"
.format(lb_config))
def test_set_config_None(self):
"""
Setting `None` via :func:`config.set_config_data` also works
and does not raise exceptions on subsequent update or get
"""
config.set_config_data(None)
self.assertIsNone(config.config_value("a"))
config.update_config_data("a.b", 2)
def when_authenticated((auth_token, service_catalog)):
log.msg('Validating launch server config')
return validate_config.validate_launch_server_config(
log,
config_value('region'),
service_catalog,
auth_token,
launch_config['args'])
def _default_throttler(locks, clock, stype, method, tenant_id):
"""
Get a throttler function with throttling policies based on configuration.
"""
cfg_name = _CFG_NAMES.get((stype, method))
if cfg_name is not None:
delay = config_value('cloud_client.throttling.' + cfg_name)
if delay is not None:
lock = locks.get_lock((stype, method))
return partial(lock.run, deferLater, clock, delay)
# Could be a per-tenant lock
cfg_name = _CFG_NAMES_PER_TENANT.get((stype, method))
if cfg_name is not None:
delay = config_value('cloud_client.throttling.' + cfg_name)
if delay is not None:
lock = locks.get_lock((stype, method, tenant_id))
return partial(lock.run, deferLater, clock, delay)
def _default_throttler(locks, clock, stype, method, tenant_id):
"""
Get a throttler function with throttling policies based on configuration.
"""
cfg_name = _CFG_NAMES.get((stype, method))
if cfg_name is not None:
delay = config_value('cloud_client.throttling.' + cfg_name)
if delay is not None:
lock = locks.get_lock((stype, method))
return partial(lock.run, deferLater, clock, delay)
# Could be a per-tenant lock
cfg_name = _CFG_NAMES_PER_TENANT.get((stype, method))
if cfg_name is not None:
delay = config_value('cloud_client.throttling.' + cfg_name)
if delay is not None:
lock = locks.get_lock((stype, method, tenant_id))
return partial(lock.run, deferLater, clock, delay)
def when_authenticated((auth_token, service_catalog)):
log.msg("Executing launch config.")
return launch_server_v1.launch_server(
log,
config_value('region'),
scaling_group,
service_catalog,
auth_token,
launch_config['args'])
def get_service_endpoint(service_catalog, region):
"""
Get the service endpoint used to connect cloud services
"""
cloudServersOpenStack = config_value('cloudServersOpenStack')
server_endpoint = public_endpoint_url(service_catalog,
cloudServersOpenStack,
region)
return server_endpoint
def test_nonexistent_value_with_shared_key_part_at_toplevel(self):
"""
:func:`~config.config_value` will return :data`None` if the path does
not exist in the nested dictionaries, even if some part of the path
does. I.e. config_value is not allowed to ignore arbitrary path
prefixes.
"""
value = config.config_value('prefix.shouldnt.be.ignored.foo')
self.assertIdentical(value, None)
def when_removed_from_loadbalancers(_ignore):
cloudServersOpenStack = config_value('cloudServersOpenStack')
server_endpoint = public_endpoint_url(request_bag.service_catalog,
cloudServersOpenStack,
request_bag.region)
return verified_delete(log,
server_endpoint,
request_bag,
server_id,
clock=clock)
def base(self, request):
"""
base root route.
:returns: Whatever is configured to be returned by the root
"""
code = config_value('root.code')
if code is not None:
request.setResponseCode(code)
headers = config_value('root.headers')
if headers is not None:
for header in headers:
for value in headers[header]:
request.setHeader(str(header), str(value))
body = config_value('root.body')
if body is not None:
return body
return ''
def remove_from_load_balancer(log, endpoint, auth_token, loadbalancer_id,
node_id, clock=None):
"""
Remove a node from a load balancer.
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone Auth Token.
:param str loadbalancer_id: The ID for a cloud loadbalancer.
:param str node_id: The ID for a node in that cloudloadbalancer.
:returns: A Deferred that fires with None if the operation completed successfully,
or errbacks with an RequestError.
"""
lb_log = log.bind(loadbalancer_id=loadbalancer_id, node_id=node_id)
# TODO: Will remove this once LB ERROR state is fixed and it is working fine
lb_log.msg('Removing from load balancer')
path = append_segments(endpoint, 'loadbalancers', str(loadbalancer_id), 'nodes', str(node_id))
def remove():
d = treq.delete(path, headers=headers(auth_token), log=lb_log)
d.addCallback(check_success, [200, 202])
d.addCallback(treq.content) # To avoid https://twistedmatrix.com/trac/ticket/6751
d.addErrback(log_lb_unexpected_errors, lb_log, 'remove_node')
d.addErrback(wrap_request_error, path, 'remove_node')
d.addErrback(check_deleted_clb, loadbalancer_id, node_id)
return d
d = retry(
remove,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(config_value('worker.lb_max_retries') or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range') or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
# A node or CLB deleted is considered successful removal
d.addErrback(lambda f: f.trap(CLBOrNodeDeleted) and lb_log.msg(f.value.message))
d.addCallback(lambda _: lb_log.msg('Removed from load balancer'))
return d
def get_collection_links(collection,
url,
rel,
limit=None,
marker=None,
next_marker=None):
"""
Return links `dict` for given collection.
The links will look somewhat like this::
[
{
"href": <url with api version>,
"rel": "self"
},
{
"href": <url of next link>,
"rel": "next"
}
]
The 'next' link is added only if number of items in `collection`
has reached `limit`.
:param collection: the collection whose links are required.
:type collection: list of dict that has 'id' in it
:param url: URL of the collection
:param rel: What to put under 'rel'
:param limit: pagination limit
:param marker: the current pagination marker
:param next_marker: a callable that takes the collection, the limit, and
the current marker, and returns the next marker
"""
if next_marker is None:
next_marker = next_marker_by_id
links = []
limit = limit or config_value('limits.pagination') or 100
if rel is not None:
links.append(_pagination_link(url, rel, limit, marker))
if len(collection) >= limit:
links.append(
_pagination_link(url, 'next', limit,
next_marker(collection, limit, marker)))
return links
def _get_policies(group):
"""
Now that we know the group exists, get its policies
"""
limit = config_value('limits.pagination') or 100
d = self._naive_list_policies(limit=limit)
d.addCallback(lambda policies: {
'groupConfiguration': _jsonloads_data(group['group_config']),
'launchConfiguration': _jsonloads_data(group['launch_config']),
'scalingPolicies': policies,
'id': self.uuid,
'state': _unmarshal_state(group)
})
return d
def create_scaling_group(self, log, tenant, config, launch, policies=None):
"""
see :meth:`otter.models.interface.IScalingGroupCollection.create_scaling_group`
"""
uuid = str(uuid4())
max_groups = config_value('limits.absolute.maxGroups')
if len(self.data[tenant]) >= max_groups:
msg = 'client has reached maxGroups limit'
log.bind(tenant_id=tenant, scaling_group_id=uuid).msg(msg)
return defer.fail(ScalingGroupOverLimitError(tenant, max_groups))
self.data[tenant][uuid] = MockScalingGroup(
log, tenant, uuid, self,
{'config': config, 'launch': launch, 'policies': policies})
return self.data[tenant][uuid].view_manifest()
def _pagination_link(url, rel, limit, marker):
"""
Generates a link dictionary where the href link has (possibly) limit
and marker query parameters, so long as they are not None.
:param url: URL of the collection
:param rel: What to put under 'rel'
:param limit: pagination limit
:param marker: the current pagination marker
:return: ``dict`` containing an href and the rel, the href being a link
to the collection represented by the url, limit, and marker
"""
query_params = {}
if marker is not None:
query_params = {'marker': marker, 'limit': limit}
elif limit != (config_value('limits.pagination') or 100):
query_params['limit'] = limit
# split_url is a tuple that can't be modified, so listify it
# (scheme, netloc, path, query, fragment)
split_url = urlsplit(url)
mutable_url_parts = list(split_url)
# update mutable_url_parts with a scheme and netloc if either are missing
# so that the final URI will always be an absolute URI
if not (split_url.scheme and split_url.netloc):
# generate a new absolute URI so that when split, its scheme, netloc,
# and path parts can be cannabalized
donor = urlsplit(
append_segments(get_url_root(), split_url.path.lstrip('/')))
mutable_url_parts[:3] = [donor.scheme, donor.netloc, donor.path]
# update the query parameters with new query parameters if necessary
if query_params:
query = parse_qs(split_url.query)
query.update(query_params)
querystring = urlencode(query, doseq=True)
# sort alphabetically for easier testing
mutable_url_parts[3] = '&'.join(sorted(querystring.split('&')))
url = urlunsplit(mutable_url_parts)
return {'href': url, 'rel': rel}
def _pagination_link(url, rel, limit, marker):
"""
Generates a link dictionary where the href link has (possibly) limit
and marker query parameters, so long as they are not None.
:param url: URL of the collection
:param rel: What to put under 'rel'
:param limit: pagination limit
:param marker: the current pagination marker
:return: ``dict`` containing an href and the rel, the href being a link
to the collection represented by the url, limit, and marker
"""
query_params = {}
if marker is not None:
query_params = {'marker': marker, 'limit': limit}
elif limit != (config_value('limits.pagination') or 100):
query_params['limit'] = limit
# split_url is a tuple that can't be modified, so listify it
# (scheme, netloc, path, query, fragment)
split_url = urlsplit(url)
mutable_url_parts = list(split_url)
# update mutable_url_parts with a scheme and netloc if either are missing
# so that the final URI will always be an absolute URI
if not (split_url.scheme and split_url.netloc):
# generate a new absolute URI so that when split, its scheme, netloc,
# and path parts can be cannabalized
donor = urlsplit(
append_segments(get_url_root(), split_url.path.lstrip('/')))
mutable_url_parts[:3] = [donor.scheme, donor.netloc, donor.path]
# update the query parameters with new query parameters if necessary
if query_params:
query = parse_qs(split_url.query)
query.update(query_params)
querystring = urlencode(query, doseq=True)
# sort alphabetically for easier testing
mutable_url_parts[3] = '&'.join(sorted(querystring.split('&')))
url = urlunsplit(mutable_url_parts)
return {'href': url, 'rel': rel}
def get_collection_links(collection, url, rel, limit=None, marker=None,
next_marker=None):
"""
Return links `dict` for given collection.
The links will look somewhat like this::
[
{
"href": <url with api version>,
"rel": "self"
},
{
"href": <url of next link>,
"rel": "next"
}
]
The 'next' link is added only if number of items in `collection`
has reached `limit`.
:param collection: the collection whose links are required.
:type collection: list of dict that has 'id' in it
:param url: URL of the collection
:param rel: What to put under 'rel'
:param limit: pagination limit
:param marker: the current pagination marker
:param next_marker: a callable that takes the collection, the limit, and
the current marker, and returns the next marker
"""
if next_marker is None:
next_marker = next_marker_by_id
links = []
limit = limit or config_value('limits.pagination') or 100
if rel is not None:
links.append(_pagination_link(url, rel, limit, marker))
if len(collection) >= limit:
links.append(_pagination_link(url, 'next', limit,
next_marker(collection, limit, marker)))
return links
def setup_scheduler(parent, store, kz_client):
"""
Setup scheduler service
"""
# Setup scheduler service
if not config_value('scheduler') or config_value('mock'):
return
buckets = range(1, int(config_value('scheduler.buckets')) + 1)
store.set_scheduler_buckets(buckets)
partition_path = config_value('scheduler.partition.path') or '/scheduler_partition'
time_boundary = config_value('scheduler.partition.time_boundary') or 15
scheduler_service = SchedulerService(int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')),
store, kz_client, partition_path, time_boundary,
buckets)
scheduler_service.setServiceParent(parent)
return scheduler_service
def get_sempahore(operation, conf_name):
"""
Get global semaphore of given operation if configured based on conf_name.
Otherwise return None
:param str operation: Operation for which semaphore is required. Must be
same each time it is called for that operation
:param str conf_name: Semaphore is returned only if this config exists
:return: A :obj:`DeferredSemaphore` object corresponding to the operation
"""
sem = _semaphores.get(operation)
if sem is not None:
return sem
conf = config_value(conf_name)
if conf is None:
return None
_semaphores[operation] = DeferredSemaphore(conf)
return _semaphores[operation]
def _(self, request, *args, **kwargs):
paginate = {}
hard_limit = config_value('limits.pagination')
if 'limit' in request.args:
try:
paginate['limit'] = int(request.args['limit'][0])
except:
return defer.fail(
InvalidQueryArgument('Invalid query argument for "limit"'))
paginate['limit'] = max(min(paginate['limit'], hard_limit), 1)
else:
paginate['limit'] = hard_limit
if 'marker' in request.args:
paginate['marker'] = request.args['marker'][0]
kwargs['paginate'] = paginate
return f(self, request, *args, **kwargs)
def _(self, request, *args, **kwargs):
paginate = {}
hard_limit = config_value('limits.pagination')
if 'limit' in request.args:
try:
paginate['limit'] = int(request.args['limit'][0])
except:
return defer.fail(InvalidQueryArgument(
'Invalid query argument for "limit"'))
paginate['limit'] = max(min(paginate['limit'], hard_limit), 1)
else:
paginate['limit'] = hard_limit
if 'marker' in request.args:
paginate['marker'] = request.args['marker'][0]
kwargs['paginate'] = paginate
return f(self, request, *args, **kwargs)
def get_sempahore(operation, conf_name):
"""
Get global semaphore of given operation if configured based on conf_name.
Otherwise return None
:param str operation: Operation for which semaphore is required. Must be
same each time it is called for that operation
:param str conf_name: Semaphore is returned only if this config exists
:return: A :obj:`DeferredSemaphore` object corresponding to the operation
"""
sem = _semaphores.get(operation)
if sem is not None:
return sem
conf = config_value(conf_name)
if conf is None:
return None
_semaphores[operation] = DeferredSemaphore(conf)
return _semaphores[operation]
def list_limits(self, request):
"""
returns application limits
"""
data = {"limits": {"absolute": config_value("limits.absolute")}}
accept = request.getHeader("accept")
if accept and 'xml' in accept:
url = "http://docs.openstack.org/common/api/v1.0"
xml = etree.Element("limits", xmlns=url)
absolute = etree.SubElement(xml, "absolute")
for key, val in data['limits']['absolute'].iteritems():
etree.SubElement(absolute, "limit", name=key, value=str(val))
request.setHeader("Content-Type", "application/xml")
return etree.tostring(xml, encoding="UTF-8", xml_declaration=True)
return json.dumps(data)
def list_limits(self, request):
"""
returns application limits
"""
data = {"limits": {"absolute": config_value("limits.absolute")}}
accept = request.getHeader("accept")
if accept and 'xml' in accept:
url = "http://docs.openstack.org/common/api/v1.0"
xml = etree.Element("limits", xmlns=url)
absolute = etree.SubElement(xml, "absolute")
for key, val in data['limits']['absolute'].iteritems():
etree.SubElement(absolute, "limit", name=key, value=str(val))
request.setHeader("Content-Type", "application/xml")
return etree.tostring(xml, encoding="UTF-8", xml_declaration=True)
return json.dumps(data)
def remove_from_load_balancer(log,
request_bag,
lb_config,
lb_response,
clock=None):
"""
Remove a node from a load balancer.
:param BoundLog log: A bound logger.
:param request_bag: A request function.
:param dict lb_config: An ``lb_config`` dictionary.
:param lb_response: The response the load balancer provided when the server
being removed was added. Type and shape is dependant on type of load
balancer.
:param IReactorTime clock: An optional clock, for testing. Will be passed
on to implementations of node removal logic for specific load balancer
APIs, if they support a clock.
:returns: A Deferred that fires with :data:`None` if the operation
completed successfully, or errbacks with an RequestError.
"""
lb_type = lb_config.get("type", "CloudLoadBalancer")
if lb_type == "CloudLoadBalancer":
cloudLoadBalancers = config_value('cloudLoadBalancers')
endpoint = public_endpoint_url(request_bag.service_catalog,
cloudLoadBalancers,
request_bag.lb_region)
auth_token = request_bag.auth_token
loadbalancer_id = lb_config["loadBalancerId"]
node_id = next(node_info["id"] for node_info in lb_response["nodes"])
return _remove_from_clb(log, endpoint, auth_token, loadbalancer_id,
node_id, clock)
elif lb_type == "RackConnectV3":
lb_id = lb_config["loadBalancerId"]
node_id = next(pair["cloud_server"]["id"] for pair in lb_response)
return remove_from_rcv3(request_bag, lb_id, node_id)
else:
raise RuntimeError(
"Unknown cloud load balancer type! config: {}".format(lb_config))
def setup_scheduler(parent, dispatcher, store, kz_client):
"""
Setup scheduler service
"""
# Setup scheduler service
if not config_value('scheduler') or config_value('mock'):
return
buckets = range(1, int(config_value('scheduler.buckets')) + 1)
store.set_scheduler_buckets(buckets)
partition_path = (config_value('scheduler.partition.path') or
'/scheduler_partition')
time_boundary = config_value('scheduler.partition.time_boundary') or 15
partitioner_factory = partial(
Partitioner,
kz_client, int(config_value('scheduler.interval')), partition_path,
buckets, time_boundary)
scheduler_service = SchedulerService(
dispatcher, int(config_value('scheduler.batchsize')),
store, partitioner_factory)
scheduler_service.setServiceParent(parent)
return scheduler_service
def when_authenticated((auth_token, service_catalog)):
log.msg("Executing launch config.")
return launch_server_v1.launch_server(log, config_value('region'),
scaling_group,
service_catalog, auth_token,
launch_config['args'], undo)
def test_top_level_value(self):
"""
:func:`~config.config_value` returns the value stored at the top level key.
"""
self.assertEqual(config.config_value('foo'), 'bar')
def launch_server(log,
request_bag,
scaling_group,
launch_config,
undo,
clock=None):
"""
Launch a new server given the launch config auth tokens and service
catalog. Possibly adding the newly launched server to a load balancer.
:param BoundLog log: A bound logger.
:param request_bag: An object with a bunch of useful data on it, including
a callable to re-auth and get a new token.
:param IScalingGroup scaling_group: The scaling group to add the launched
server to.
:param dict launch_config: A launch_config args structure as defined for
the launch_server_v1 type.
:param IUndoStack undo: The stack that will be rewound if undo fails.
:return: Deferred that fires with a 2-tuple of server details and the
list of load balancer responses from add_to_load_balancers.
"""
launch_config = prepare_launch_config(scaling_group.uuid, launch_config)
cloudServersOpenStack = config_value('cloudServersOpenStack')
server_endpoint = public_endpoint_url(request_bag.service_catalog,
cloudServersOpenStack,
request_bag.region)
lb_config = launch_config.get('loadBalancers', [])
server_config = launch_config['server']
log = log.bind(server_name=server_config['name'])
ilog = [None]
def check_metadata(server):
# sanity check to make sure the metadata didn't change - can probably
# be removed after a while if we do not see any log messages from this
# function
expected = launch_config['server']['metadata']
result = server['server'].get('metadata')
if result != expected:
ilog[0].msg('Server metadata has changed.',
sanity_check=True,
expected_metadata=expected,
nova_metadata=result)
return server
def wait_for_server(server, new_request_bag):
server_id = server['server']['id']
# NOTE: If server create is retried, each server delete will be pushed
# to undo stack even after it will be deleted in check_error which is
# fine since verified_delete succeeds on deleted server
undo.push(verified_delete, log, server_endpoint, new_request_bag,
server_id)
ilog[0] = log.bind(server_id=server_id)
return wait_for_active(ilog[0], server_endpoint,
new_request_bag.auth_token,
server_id).addCallback(check_metadata)
def add_lb(server, new_request_bag):
if lb_config:
lbd = add_to_load_balancers(ilog[0], new_request_bag, lb_config,
server, undo)
lbd.addCallback(lambda lb_response: (server, lb_response))
return lbd
return (server, [])
def _real_create_server(new_request_bag):
auth_token = new_request_bag.auth_token
d = create_server(server_endpoint, auth_token, server_config, log=log)
d.addCallback(wait_for_server, new_request_bag)
d.addCallback(add_lb, new_request_bag)
return d
def _create_server():
return request_bag.re_auth().addCallback(_real_create_server)
def check_error(f):
f.trap(UnexpectedServerStatus)
if f.value.status == 'ERROR':
log.msg(
'{server_id} errored, deleting and creating new '
'server instead',
server_id=f.value.server_id)
# trigger server delete and return True to allow retry
verified_delete(log, server_endpoint, request_bag,
f.value.server_id)
return True
else:
return False
d = retry(_create_server,
can_retry=compose_retries(retry_times(3), check_error),
next_interval=repeating_interval(15),
clock=clock)
return d
def test_nested_value(self):
"""
:func:`~config.config_value` returns the value stored at a . separated
path.
"""
self.assertEqual(config.config_value('baz.bax'), 'quux')
def add_to_clb(log,
endpoint,
auth_token,
lb_config,
ip_address,
undo,
clock=None):
"""
Add an IP address to a Cloud Load Balancer based on the ``lb_config``.
TODO: Handle load balancer node metadata.
:param log: A bound logger
:param str endpoint: Load balancer endpoint URI.
:param str auth_token: Keystone auth token.
:param dict lb_config: An ``lb_config`` dictionary.
:param str ip_address: The IP address of the node to add to the load
balancer.
:param IUndoStack undo: An IUndoStack to push any reversable operations
onto.
:return: Deferred that fires with the load balancer response.
"""
lb_id = lb_config['loadBalancerId']
port = lb_config['port']
path = append_segments(endpoint, 'loadbalancers', str(lb_id), 'nodes')
lb_log = log.bind(loadbalancer_id=lb_id, ip_address=ip_address)
def add():
d = treq.post(path,
headers=headers(auth_token),
data=json.dumps({
"nodes": [{
"address": ip_address,
"port": port,
"condition": "ENABLED",
"type": "PRIMARY"
}]
}),
log=lb_log)
d.addCallback(check_success, [200, 202])
d.addErrback(log_lb_unexpected_errors, lb_log, 'add_node')
d.addErrback(wrap_request_error, path, 'add_node')
d.addErrback(check_deleted_clb, lb_id)
return d
d = retry(add,
can_retry=compose_retries(
transient_errors_except(CLBOrNodeDeleted),
retry_times(
config_value('worker.lb_max_retries')
or LB_MAX_RETRIES)),
next_interval=random_interval(
*(config_value('worker.lb_retry_interval_range')
or LB_RETRY_INTERVAL_RANGE)),
clock=clock)
def when_done(result):
node_id = result['nodes'][0]['id']
lb_log.msg('Added to load balancer', node_id=node_id)
undo.push(_remove_from_clb, lb_log, endpoint, auth_token, lb_id,
node_id)
return result
return d.addCallback(treq.json_content).addCallback(when_done)
def test_non_existent_value(self):
"""
:func:`~config.config_value` will return :data`None` if the path does
not exist in the nested dictionaries.
"""
self.assertIdentical(config.config_value('baz.blah'), None)
def when_authenticated((auth_token, service_catalog)):
log.msg('Deleting server')
return launch_server_v1.delete_server(
log, config_value('region'), service_catalog, auth_token,
(server['id'], server['lb_info']))
def makeService(config):
"""
Set up the otter-api service.
"""
set_config_data(dict(config))
if not config_value('mock'):
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')
]
cassandra_cluster = LoggingCQLClient(
RoundRobinCassandraCluster(seed_endpoints,
config_value('cassandra.keyspace')),
log.bind(system='otter.silverberg'))
set_store(CassScalingGroupCollection(cassandra_cluster))
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
cache_ttl = config_value('identity.cache_ttl')
if cache_ttl is None:
# FIXME: Pick an arbitrary cache ttl value based on absolutely no
# science.
cache_ttl = 300
authenticator = CachingAuthenticator(
reactor,
ImpersonatingAuthenticator(config_value('identity.username'),
config_value('identity.password'),
config_value('identity.url'),
config_value('identity.admin_url')),
cache_ttl)
supervisor = Supervisor(authenticator.authenticate_tenant, coiterate)
set_supervisor(supervisor)
s = MultiService()
site = Site(root)
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(s)
if config_value('scheduler') and not config_value('mock'):
scheduler_service = SchedulerService(
int(config_value('scheduler.batchsize')),
int(config_value('scheduler.interval')), cassandra_cluster)
scheduler_service.setServiceParent(s)
return s
def get_url_root():
"""
Get the URL root
:return: string containing the URL root
"""
return config_value('url_root')
def makeService(config):
"""
Set up the otter-api service.
"""
config = dict(config)
set_config_data(config)
parent = MultiService()
region = config_value('region')
seed_endpoints = [
clientFromString(reactor, str(host))
for host in config_value('cassandra.seed_hosts')]
cassandra_cluster = LoggingCQLClient(
TimingOutCQLClient(
reactor,
RoundRobinCassandraCluster(
seed_endpoints,
config_value('cassandra.keyspace'),
disconnect_on_cancel=True),
config_value('cassandra.timeout') or 30),
log.bind(system='otter.silverberg'))
store = CassScalingGroupCollection(
cassandra_cluster, reactor, config_value('limits.absolute.maxGroups'))
admin_store = CassAdmin(cassandra_cluster)
bobby_url = config_value('bobby_url')
if bobby_url is not None:
set_bobby(BobbyClient(bobby_url))
service_configs = get_service_configs(config)
authenticator = generate_authenticator(reactor, config['identity'])
supervisor = SupervisorService(authenticator, region, coiterate,
service_configs)
supervisor.setServiceParent(parent)
set_supervisor(supervisor)
health_checker = HealthChecker(reactor, {
'store': getattr(store, 'health_check', None),
'kazoo': store.kazoo_health_check,
'supervisor': supervisor.health_check
})
# Setup cassandra cluster to disconnect when otter shuts down
if 'cassandra_cluster' in locals():
parent.addService(FunctionalService(stop=partial(
call_after_supervisor, cassandra_cluster.disconnect, supervisor)))
otter = Otter(store, region, health_checker.health_check)
site = Site(otter.app.resource())
site.displayTracebacks = False
api_service = service(str(config_value('port')), site)
api_service.setServiceParent(parent)
# Setup admin service
admin_port = config_value('admin')
if admin_port:
admin = OtterAdmin(admin_store)
admin_site = Site(admin.app.resource())
admin_site.displayTracebacks = False
admin_service = service(str(admin_port), admin_site)
admin_service.setServiceParent(parent)
# setup cloud feed
cf_conf = config.get('cloudfeeds', None)
if cf_conf is not None:
id_conf = deepcopy(config['identity'])
id_conf['strategy'] = 'single_tenant'
add_to_fanout(CloudFeedsObserver(
reactor=reactor,
authenticator=generate_authenticator(reactor, id_conf),
tenant_id=cf_conf['tenant_id'],
region=region,
service_configs=service_configs))
# Setup Kazoo client
if config_value('zookeeper'):
threads = config_value('zookeeper.threads') or 10
disable_logs = config_value('zookeeper.no_logs')
threadpool = ThreadPool(maxthreads=threads)
sync_kz_client = KazooClient(
hosts=config_value('zookeeper.hosts'),
# Keep trying to connect until the end of time with
# max interval of 10 minutes
connection_retry=dict(max_tries=-1, max_delay=600),
logger=None if disable_logs else TxLogger(log.bind(system='kazoo'))
)
kz_client = TxKazooClient(reactor, threadpool, sync_kz_client)
# Don't timeout. Keep trying to connect forever
d = kz_client.start(timeout=None)
def on_client_ready(_):
dispatcher = get_full_dispatcher(reactor, authenticator, log,
get_service_configs(config),
kz_client, store, supervisor,
cassandra_cluster)
# Setup scheduler service after starting
scheduler = setup_scheduler(parent, dispatcher, store, kz_client)
health_checker.checks['scheduler'] = scheduler.health_check
otter.scheduler = scheduler
# Give dispatcher to Otter REST object
otter.dispatcher = dispatcher
# Set the client after starting
# NOTE: There is small amount of time when the start is
# not finished and the kz_client is not set in which case
# policy execution and group delete will fail
store.kz_client = kz_client
# Setup kazoo to stop when shutting down
parent.addService(FunctionalService(
stop=partial(call_after_supervisor,
kz_client.stop, supervisor)))
setup_converger(
parent, kz_client, dispatcher,
config_value('converger.interval') or 10,
config_value('converger.build_timeout') or 3600,
config_value('converger.limited_retry_iterations') or 10,
config_value('converger.step_limits') or {})
d.addCallback(on_client_ready)
d.addErrback(log.err, 'Could not start TxKazooClient')
return parent