def _assess_interface(interface, optional, missing_interfaces,
incomplete_interfaces):
"""Assess a named interface for presence and completeness
Uses reactive flags 'connected' and 'available' to indicate whether
an interface is present and complete.
:param: interface: Name of interface to assess.
:param: options: Boolean indicating whether interface is optional
:param: missing_interfaces: List of missing interfaces to update
:param: incomplete_interfaces: List of incomplete interfaces to update
:returns: bool, bool: Tuple of booleans indicating (missing, incomplete)
"""
log("Assessing interface {}".format(interface), level=DEBUG)
base_name = interface.split('.')[0]
connected = (is_flag_set('{}.connected'.format(interface))
or is_flag_set('{}.connected'.format(base_name)))
missing = False
incomplete = False
if not connected:
if not optional:
missing_interfaces.append(base_name)
missing = True
incomplete = True
elif connected and not is_flag_set('{}.available'.format(interface)):
incomplete_interfaces.append(base_name)
incomplete = True
return (missing, incomplete)
def publish_global_client_cert():
"""
This is for backwards compatibility with older tls-certificate clients
only. Obviously, it's not good security / design to have clients sharing
a certificate, but it seems that there are clients that depend on this
(though some, like etcd, only block on the flag that it triggers but don't
actually use the cert), so we have to set it for now.
"""
if not client_approle_authorized():
log("Vault not authorized: Skipping publish_global_client_cert",
"WARNING")
return
cert_created = is_flag_set('charm.vault.global-client-cert.created')
reissue_requested = is_flag_set('certificates.reissue.global.requested')
tls = endpoint_from_flag('certificates.available')
if not cert_created or reissue_requested:
ttl = config()['default-ttl']
max_ttl = config()['max-ttl']
bundle = vault_pki.generate_certificate('client', 'global-client', [],
ttl, max_ttl)
unitdata.kv().set('charm.vault.global-client-cert', bundle)
set_flag('charm.vault.global-client-cert.created')
clear_flag('certificates.reissue.global.requested')
else:
bundle = unitdata.kv().get('charm.vault.global-client-cert')
tls.set_client_cert(bundle['certificate'], bundle['private_key'])
def _assess_status():
"""Assess status of relations and services for local unit"""
if is_flag_set('snap.channel.invalid'):
status_set('blocked',
'Invalid snap channel '
'configured: {}'.format(config('channel')))
return
if is_flag_set('config.dns_vip.invalid'):
status_set('blocked',
'vip and dns-ha-access-record configured')
return
health = None
if service_running('vault'):
health = vault.get_vault_health()
application_version_set(health.get('version'))
_missing_interfaces = []
_incomplete_interfaces = []
_assess_interface_groups(REQUIRED_INTERFACES, optional=False,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
_assess_interface_groups(OPTIONAL_INTERFACES, optional=True,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
if _missing_interfaces or _incomplete_interfaces:
state = 'blocked' if _missing_interfaces else 'waiting'
status_set(state, ', '.join(_missing_interfaces +
_incomplete_interfaces))
return
if not service_running('vault'):
status_set('blocked', 'Vault service not running')
return
if not health['initialized']:
status_set('blocked', 'Vault needs to be initialized')
return
if health['sealed']:
status_set('blocked', 'Unit is sealed')
return
mlock_disabled = is_container() or config('disable-mlock')
status_set(
'active',
'Unit is ready '
'(active: {}, mlock: {})'.format(
str(not health['standby']).lower(),
'disabled' if mlock_disabled else 'enabled'
)
)
def test_when_not_set_clear(self):
flags.register_trigger(when_not='foo',
set_flag='bar',
clear_flag='qux')
flags.set_flag('noop')
flags.clear_flag('noop')
assert not flags.is_flag_set('bar')
flags.set_flag('foo')
flags.set_flag('qux')
assert not flags.is_flag_set('bar')
flags.clear_flag('foo')
assert flags.is_flag_set('bar')
assert not flags.is_flag_set('qux')
def configure_vault(context):
log("Running configure_vault", level=DEBUG)
context['disable_mlock'] = is_container() or config('disable-mlock')
context['ssl_available'] = is_state('vault.ssl.available')
if is_flag_set('etcd.tls.available'):
etcd = endpoint_from_flag('etcd.available')
log("Etcd detected, adding to context", level=DEBUG)
context['etcd_conn'] = etcd.connection_string()
context['etcd_tls_ca_file'] = '/var/snap/vault/common/etcd-ca.pem'
context['etcd_tls_cert_file'] = '/var/snap/vault/common/etcd-cert.pem'
context['etcd_tls_key_file'] = '/var/snap/vault/common/etcd.key'
save_etcd_client_credentials(etcd,
key=context['etcd_tls_key_file'],
cert=context['etcd_tls_cert_file'],
ca=context['etcd_tls_ca_file'])
context['api_addr'] = vault.get_api_url()
context['cluster_addr'] = vault.get_cluster_url()
log("Etcd detected, setting api_addr to {}".format(
context['api_addr']))
else:
log("Etcd not detected", level=DEBUG)
log("Rendering vault.hcl.j2", level=DEBUG)
render('vault.hcl.j2', VAULT_CONFIG, context, perms=0o600)
log("Rendering vault systemd configuation", level=DEBUG)
render('vault.service.j2', VAULT_SYSTEMD_CONFIG, {}, perms=0o644)
service('enable', 'vault')
log("Opening vault port", level=DEBUG)
open_port(8200)
set_flag('configured')
if any_file_changed([VAULT_CONFIG, VAULT_SYSTEMD_CONFIG]):
# force a restart if config has changed
clear_flag('started')
def configure_node(cluster_changed, cluster_joined):
status_set('maintenance', 'Configuring slurm-node')
controller_data = cluster_changed.active_data
create_spool_dir(context=controller_data)
render_munge_key(context=controller_data)
# If the munge.key has been changed on the controller and munge is
# running, the service must be restarted to use the new key
if flags.is_flag_set('endpoint.slurm-cluster.changed.munge_key'
) and service_running(MUNGE_SERVICE):
log('Restarting munge due to key change on slurm-controller')
service_restart(MUNGE_SERVICE)
render_slurm_config(context=controller_data)
# Make sure munge is running
if not service_running(MUNGE_SERVICE):
service_start(MUNGE_SERVICE)
# Make sure slurmd is running
if not service_running(SLURMD_SERVICE):
service_start(SLURMD_SERVICE)
flags.set_flag('slurm-node.configured')
log('Set {} flag'.format('slurm-node.configured'))
flags.clear_flag('endpoint.slurm-cluster.active.changed')
log('Cleared {} flag'.format('endpoint.slurm-cluster.active.changed'))
# Clear this flag to be able to signal munge_key changed if it occurs from
# a controller.
flags.clear_flag('endpoint.slurm-cluster.changed.munge_key')
log('Cleared {} flag'.format('endpoint.slurm-cluster.changed.munge_key'))
def send_vault_url_and_ca():
secrets = endpoint_from_flag('secrets.connected')
vault_url_external = None
if is_flag_set('ha.available'):
hostname = config('hostname')
if hostname:
vault_url = vault.get_api_url(address=hostname)
else:
vip = vault.get_vip()
vault_url = vault.get_api_url(address=vip)
ext_vip = vault.get_vip(binding='external')
if ext_vip and ext_vip != vip:
vault_url_external = vault.get_api_url(address=ext_vip,
binding='external')
else:
vault_url = vault.get_api_url()
vault_url_external = vault.get_api_url(binding='external')
if vault_url_external == vault_url:
vault_url_external = None
secrets.publish_url(vault_url=vault_url, remote_binding='access')
if vault_url_external:
secrets.publish_url(vault_url=vault_url_external,
remote_binding='external')
if config('ssl-ca'):
secrets.publish_ca(vault_ca=config('ssl-ca'))
def create_certs():
reissue_requested = is_flag_set('certificates.reissue.requested')
tls = endpoint_from_flag('certificates.available')
requests = tls.all_requests if reissue_requested else tls.new_requests
if reissue_requested:
log('Reissuing all certs')
processed_applications = []
for request in requests:
log('Processing certificate request from {} for {}'.format(
request.unit_name, request.common_name))
if request.cert_type == 'application':
cert_type = 'server'
# When an application cert is published all units recieve the same
# data so one need to process one request per application.
if request.application_name in processed_applications:
log('Already done {}'.format(request.application_name))
continue
else:
processed_applications.append(request.application_name)
else:
cert_type = request.cert_type
try:
ttl = config()['default-ttl']
max_ttl = config()['max-ttl']
bundle = vault_pki.generate_certificate(cert_type,
request.common_name,
request.sans, ttl, max_ttl)
request.set_cert(bundle['certificate'], bundle['private_key'])
except vault.VaultInvalidRequest as e:
log(str(e), level=ERROR)
continue # TODO: report failure back to client
clear_flag('certificates.reissue.requested')
def _manage_flags(self):
"""
Manage automatic relation flags.
Internal use only.
"""
already_joined = is_flag_set(self.expand_name('joined'))
hook_name = hookenv.hook_name()
rel_hook = hook_name.startswith(self.endpoint_name + '-relation-')
departed_hook = rel_hook and hook_name.endswith('-departed')
toggle_flag(self.expand_name('joined'), self.is_joined)
if departed_hook:
set_flag(self.expand_name('departed'))
elif self.is_joined:
clear_flag(self.expand_name('departed'))
if already_joined and not rel_hook:
# skip checking relation data outside hooks for this relation
# to save on API calls to the controller (unless we didn't have
# the joined flag before, since then we might migrating to Endpoints)
return
for unit in self.all_units:
for key, value in unit.received.items():
data_key = 'endpoint.{}.{}.{}.{}'.format(
self.endpoint_name, unit.relation.relation_id,
unit.unit_name, key)
if data_changed(data_key, value):
set_flag(self.expand_name('changed'))
set_flag(self.expand_name('changed.{}'.format(key)))
self.manage_flags()
def send_vault_url_and_ca():
secrets = endpoint_from_flag('secrets.connected')
vault_url_external = None
hostname = config('hostname')
vip = vault.get_vip()
if is_flag_set('ha.available'):
if hostname:
vault_url = vault.get_api_url(address=hostname)
else:
vault_url = vault.get_api_url(address=vip)
ext_vip = vault.get_vip(binding='external')
if ext_vip and ext_vip != vip:
vault_url_external = vault.get_api_url(address=ext_vip,
binding='external')
elif vip:
log(
"VIP is set but ha.available is not yet set, skipping "
"send_vault_url_and_ca.",
level=DEBUG)
return
else:
vault_url = vault.get_api_url()
vault_url_external = vault.get_api_url(binding='external')
if vault_url_external == vault_url:
vault_url_external = None
secrets.publish_url(vault_url=vault_url, remote_binding='access')
if vault_url_external:
secrets.publish_url(vault_url=vault_url_external,
remote_binding='external')
if config('ssl-ca'):
secrets.publish_ca(vault_ca=config('ssl-ca'))
def send_vault_url_and_ca():
secrets = endpoint_from_flag('secrets.connected')
if is_flag_set('ha.available'):
vault_url = vault.get_api_url(address=config('vip'))
else:
vault_url = vault.get_api_url()
secrets.publish_url(vault_url=vault_url)
if config('ssl-ca'):
secrets.publish_ca(vault_ca=config('ssl-ca'))
def test_triggers(self, kv):
kv.return_value = MockKV()
assert not flags.any_flags_set('foo', 'bar', 'qux')
flags.set_flag('foo')
assert not flags.any_flags_set('bar', 'qux')
flags.clear_flag('foo')
assert not flags.any_flags_set('foo', 'bar', 'qux')
flags.register_trigger(when='foo', set_flag='bar')
flags.set_flag('foo')
assert flags.is_flag_set('bar')
flags.clear_flag('foo')
assert flags.is_flag_set('bar')
flags.clear_flag('bar')
flags.register_trigger(when='foo', clear_flag='qux')
flags.set_flag('qux')
flags.set_flag('foo')
assert not flags.is_flag_set('qux')
def create_certs():
reissue_requested = is_flag_set('certificates.reissue.requested')
tls = endpoint_from_flag('certificates.available')
requests = tls.all_requests if reissue_requested else tls.new_requests
if reissue_requested:
log('Reissuing all certs')
for request in requests:
log('Processing certificate request from {} for {}'.format(
request.unit_name, request.common_name))
try:
bundle = vault_pki.generate_certificate(request.cert_type,
request.common_name,
request.sans)
request.set_cert(bundle['certificate'], bundle['private_key'])
except vault.VaultInvalidRequest as e:
log(str(e), level=ERROR)
continue # TODO: report failure back to client
clear_flag('certificates.reissue.requested')
def send_vault_url_and_ca():
secrets = endpoint_from_flag('secrets.connected')
lb_provider = endpoint_from_name('lb-provider')
vault_url_external = None
hostname = config('hostname')
vip = vault.get_vip()
if is_flag_set('ha.available'):
if hostname:
vault_url = vault.get_api_url(address=hostname)
else:
vault_url = vault.get_api_url(address=vip)
ext_vip = vault.get_vip(binding='external')
if ext_vip and ext_vip != vip:
vault_url_external = vault.get_api_url(address=ext_vip,
binding='external')
elif vip:
log(
"VIP is set but ha.available is not yet set, skipping "
"send_vault_url_and_ca.",
level=DEBUG)
return
elif lb_provider.has_response:
response = lb_provider.get_response('vault')
if response.error:
log('Load balancer failed, skipping: '
'{}'.format(response.error_message or response.error_fields),
level=ERROR)
return
vault_url = vault.get_api_url(address=response.address)
vault_url_external = vault_url
lb_provider.ack_response(response)
else:
vault_url = vault.get_api_url()
vault_url_external = vault.get_api_url(binding='external')
if vault_url_external == vault_url:
vault_url_external = None
secrets.publish_url(vault_url=vault_url, remote_binding='access')
if vault_url_external:
secrets.publish_url(vault_url=vault_url_external,
remote_binding='external')
if config('ssl-ca'):
secrets.publish_ca(vault_ca=config('ssl-ca'))
def from_flag(cls, flag):
"""
Return an Endpoint subclass instance based on the given flag.
The instance that is returned depends on the endpoint name embedded
in the flag. Flags should be of the form ``endpoint.{name}.extra...``,
though for legacy purposes, the ``endpoint.`` prefix can be omitted.
The ``{name}}`` portion will be passed to
:meth:`~charms.reactive.endpoints.Endpoint.from_name`.
If the flag is not set, an appropriate Endpoint subclass cannot be
found, or the flag name can't be parsed, ``None`` will be returned.
"""
if not is_flag_set(flag) or '.' not in flag:
return None
parts = flag.split('.')
if parts[0] == 'endpoint':
return cls.from_name(parts[1])
else:
# some older handlers might not use the 'endpoint' prefix
return cls.from_name(parts[0])
def included_config_changed():
# Seems it is not possible to have a @when('leadership.is_leader') together with when_file_changed()
if flags.is_flag_set('leadership.is_leader'):
hookenv.log('Should restart slurmctld due to changed file %s/slurm-%s.conf on disk' %
(helpers.SLURM_CONFIG_DIR, hookenv.config().get('clustername')))
flags.set_flag('slurm-controller.reconfigure')
def upgrade_charm():
status_set('maintenance', 'Upgrading charm..')
if is_flag_set('cert-created'):
clear_flag('cert-created')
set_flag('client.cert-created')
def message():
'''Set a message to notify the user that this charm is ready.'''
if is_flag_set('client.available'):
status_set('active', 'Certificate Authority connected.')
else:
status_set('active', 'Certificate Authority ready.')
def _assess_status():
"""Assess status of relations and services for local unit"""
if is_flag_set('snap.channel.invalid'):
status_set(
'blocked', 'Invalid snap channel '
'configured: {}'.format(config('channel')))
return
if is_flag_set('config.dns_vip.invalid'):
status_set('blocked', 'vip and dns-ha-access-record configured')
return
if is_flag_set('config.lb_vip.invalid'):
status_set('blocked', 'lb-provider and vip are mutually exclusive')
return
if is_flag_set('config.lb_dns.invalid'):
status_set(
'blocked', 'lb-provider and dns-ha-access-record are '
'mutually exclusive')
return
if unitdata.kv().get('charm.vault.series-upgrading'):
status_set(
"blocked", "Ready for do-release-upgrade and reboot. "
"Set complete when finished.")
return
if is_flag_set('failed.to.start'):
status_set("blocked",
"Vault failed to start; check journalctl -u vault")
return
_missing_interfaces = []
_incomplete_interfaces = []
_assess_interface_groups(REQUIRED_INTERFACES,
optional=False,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
if _missing_interfaces or _incomplete_interfaces:
state = 'blocked' if _missing_interfaces else 'waiting'
status_set(state,
', '.join(_missing_interfaces + _incomplete_interfaces))
return
health = None
if service_running('vault'):
try:
health = vault.get_vault_health()
except Exception:
log(traceback.format_exc(), level=ERROR)
status_set('blocked', 'Vault health check failed')
return
else:
status_set('blocked', 'Vault service not running')
return
if health.get('version'):
application_version_set(health.get('version'))
else:
application_version_set('Unknown')
status_set('blocked', 'Unknown vault version')
return
if not health['initialized']:
status_set('blocked', 'Vault needs to be initialized')
return
if health['sealed']:
status_set('blocked', 'Unit is sealed')
return
if not leader_get(vault.CHARM_ACCESS_ROLE_ID):
status_set(
'blocked',
'Vault charm not yet authorized: run authorize-charm action.')
return
if not client_approle_authorized():
status_set('blocked', 'Vault cannot authorize approle')
return
lb_provider = endpoint_from_name('lb-provider')
is_leader = is_flag_set('leadership.is_leader')
if is_leader and lb_provider and lb_provider.is_available:
if not lb_provider.has_response:
status_set('waiting', 'Waiting for load balancer')
return
response = lb_provider.get_response('vault')
if response.error:
status_set(
'blocked', 'Load balancer failed: '
'{}'.format(response.error_message or response.error_fields))
return
is_leader = is_flag_set('leadership.is_leader')
has_ca = is_flag_set('charm.vault.ca.ready')
has_cert_reqs = is_flag_set('certificates.certs.requested')
if is_leader and has_cert_reqs and not has_ca:
status_set('blocked', 'Missing CA cert')
return
has_certs_relation = is_flag_set('certificates.available')
if is_leader and has_certs_relation and not has_ca:
status_set('blocked', 'Missing CA cert')
return
_assess_interface_groups(OPTIONAL_INTERFACES,
optional=True,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
if _missing_interfaces or _incomplete_interfaces:
state = 'blocked' if _missing_interfaces else 'waiting'
status_set(state,
', '.join(_missing_interfaces + _incomplete_interfaces))
return
mlock_disabled = is_container() or config('disable-mlock')
vault_installed_version = snap.get_installed_version('vault')
vault_running_version = health.get('version')
if vault_installed_version != vault_running_version:
status_set(
'active',
'New version of vault installed, manual intervention required '
'to restart the service.')
return
if is_flag_set('etcd.tls.available'):
client = vault.get_local_client()
if not client.ha_status['ha_enabled']:
status_set(
'active',
'Vault running as non-HA, manual intervention required '
'to restart the service.')
return
status_set(
'active', 'Unit is ready '
'(active: {}, mlock: {})'.format(
str(not health['standby']).lower(),
'disabled' if mlock_disabled else 'enabled'))
def test_when_set(self):
flags.register_trigger(when='foo', set_flag='bar')
flags.set_flag('foo')
assert flags.is_flag_set('bar')
flags.clear_flag('foo')
assert flags.is_flag_set('bar')
def test_when_clear(self):
flags.register_trigger(when='foo', clear_flag='qux')
flags.set_flag('qux')
flags.set_flag('foo')
assert not flags.is_flag_set('qux')
def configure_controller(*args):
''' A controller is only configured after leader election is
performed. Cluster endpoint must be present for a controller to
proceed with initial configuration'''
hookenv.status_set('maintenance', 'Configuring slurm-controller')
flags.clear_flag('slurm-controller.configured')
# need to have a role determined here so that a controller context can
# be uniformly prepared for consumption on the worker side as controller
# and node layers share a common layer with a slurm.conf template
# mostly identical on all nodes
is_active = controller.is_active_controller()
role = controller.ROLES[is_active]
peer_role = controller.ROLES[not is_active]
# the endpoint is present as joined is required for this handler
cluster_endpoint = relations.endpoint_from_flag(
'endpoint.slurm-cluster.joined')
# Get node configs
nodes = cluster_endpoint.get_node_data()
partitions = controller.get_partitions(nodes)
# Implementation of automatic node weights
node_weight_criteria = hookenv.config().get('node_weight_criteria')
if node_weight_criteria != 'none':
weightres = controller.set_node_weight_criteria(node_weight_criteria, nodes)
# If the weight configuration is incorrect, abort reconfiguration. Status
# will be set to blocked with an informative message. The controller charm
# will keep running.
if not weightres:
return
# relation-changed does not necessarily mean that data will be provided
if not partitions:
flags.clear_flag('endpoint.slurm-cluster.changed')
return
# the whole charm config will be sent to related nodes
# with some additional options added via dict update
controller_conf = copy.deepcopy(hookenv.config())
# if controller cluster config include file exists, add contents to controller_conf dict
slurmconf_include = '%s/slurm-%s.conf' % (helpers.SLURM_CONFIG_DIR, hookenv.config().get('clustername'))
if os.path.exists(slurmconf_include):
f = open(slurmconf_include, "r")
controller_conf.update({'include': f.read()})
f.close()
controller_conf.update({
'nodes': nodes,
'partitions': partitions,
# for worker nodes
'munge_key': hookenv.leader_get('munge_key'),
})
net_details = controller.add_key_prefix(
cluster_endpoint.network_details(), role)
# update the config dict used as a context in rendering to have prefixed
# keys for network details based on a current unit role (active or backup)
controller_conf.update(net_details)
ha_endpoint = relations.endpoint_from_flag(
'endpoint.slurm-controller-ha.joined')
if ha_endpoint:
# add prefixed peer data
peer_data = controller.add_key_prefix(
ha_endpoint.peer_data, peer_role)
controller_conf.update(peer_data)
else:
peer_data = None
# If we have a DBD relation, extract endpoint data and configure DBD setup
# directly, regardless if the clustername gets accepted in the DBD or not
if flags.is_flag_set('endpoint.slurm-dbd-consumer.joined') and leadership.leader_get('dbd_host'):
dbd_host = leadership.leader_get('dbd_host')
controller_conf.update({
'dbd_host': leadership.leader_get('dbd_host'),
'dbd_port': leadership.leader_get('dbd_port'),
'dbd_ipaddr': leadership.leader_get('dbd_ipaddr')
})
es_endpoint = relations.endpoint_from_flag(
'elasticsearch.available')
if es_endpoint:
for unit in es_endpoint.list_unit_data():
elastic_host = unit['host']
elastic_port = unit['port']
controller_conf.update({
'elastic_host': elastic_host,
'elastic_port': elastic_port,
})
hookenv.log(("elasticsearch available, using %s:%s from endpoint relation.") % (elastic_host,elastic_port))
else:
hookenv.log('No endpoint for elasticsearch available')
# In case we are here due to DBD join or charm config change, announce this to the nodes
# by changing the value of slurm_config_updated
if flags.is_flag_set('slurm.dbd_host_updated') or flags.is_flag_set('config.changed'):
ts = time.time()
hookenv.log('Slurm configuration on controller was updated on %s, annoucing to nodes' % ts)
controller_conf.update({ 'slurm_config_updated': ts })
flags.clear_flag('slurm.dbd_host_updated')
# a controller service is configurable if it is an active controller
# or a backup controller that knows about an active controller
is_configurable = is_active or (not is_active and peer_data)
if is_configurable:
hookenv.log('The controller is configurable ({})'.format(role))
# Setup slurm dirs and config
helpers.create_state_save_location(context=controller_conf)
helpers.render_slurm_config(context=controller_conf, active_controller=is_active)
flags.set_flag('slurm-controller.configured')
flags.clear_flag('slurm-controller.reconfigure')
flags.clear_flag('slurm-controller.munge_updated')
# restart controller process on any changes
# TODO: this could be optimized via goal-state hook by
# observing "joining" node units
host.service_restart(helpers.SLURMCTLD_SERVICE)
else:
hookenv.log('The controller is NOT configurable ({})'.format(role))
if not is_active:
hookenv.status_set('maintenance',
'Backup controller is waiting for peer data')
# Send config to nodes
if is_active:
# TODO: wait until a peer acknowledges that it has cleared
# its side of a node-facing relation - this needs to be done
# in case an active controller is changed to a different one
# to avoid split-brain conditions on node units
cluster_endpoint.send_controller_config(controller_conf)
else:
# otherwise make sure that all keys are cleared
# this is relevant for a former active controller
cluster_endpoint.send_controller_config({
k: None for k in controller_conf.keys()
})
# clear the changed flag as it is not cleared automatically
flags.clear_flag('endpoint.slurm-cluster.changed')
def configure_secrets_backend():
""" Process requests for setup and access to simple kv secret backends """
@tenacity.retry(wait=tenacity.wait_exponential(multiplier=1, max=10),
stop=tenacity.stop_after_attempt(10),
reraise=True)
def _check_vault_status(client):
if (not service_running('vault') or not client.is_initialized()
or client.is_sealed()):
return False
return True
# NOTE: use localhost listener as policy only allows 127.0.0.1 to
# administer the local vault instances via the charm
client = vault.get_client(url=vault.VAULT_LOCALHOST_URL)
status_ok = _check_vault_status(client)
if not status_ok:
log(
'Unable to process new secret backend requests,'
' deferring until vault is fully configured',
level=DEBUG)
return
charm_role_id = vault.get_local_charm_access_role_id()
if charm_role_id is None:
log(
'Charm access to vault not configured, deferring'
' secrets backend setup',
level=DEBUG)
return
client.auth_approle(charm_role_id)
secrets = (endpoint_from_flag('endpoint.secrets.new-request')
or endpoint_from_flag('secrets.connected'))
requests = secrets.requests()
# Configure KV secret backends
backends = set([request['secret_backend'] for request in requests])
for backend in backends:
if not backend.startswith('charm-'):
continue
vault.configure_secret_backend(client, name=backend)
refresh_secrets = is_flag_set('secrets.refresh')
# Configure AppRoles for application unit access
for request in requests:
# NOTE: backends must start with charm-
backend_name = request['secret_backend']
if not backend_name.startswith('charm-'):
continue
unit = request['unit']
hostname = request['hostname']
access_address = request['ingress_address']
isolated = request['isolated']
unit_name = request.get('unit_name', unit.unit_name).replace('/', '-')
policy_name = approle_name = 'charm-{}'.format(unit_name)
if isolated:
policy_template = vault.SECRET_BACKEND_HCL
else:
policy_template = vault.SECRET_BACKEND_SHARED_HCL
vault.configure_policy(client,
name=policy_name,
hcl=policy_template.format(backend=backend_name,
hostname=hostname))
cidr = '{}/32'.format(access_address)
new_role = (approle_name not in client.list_roles())
approle_id = vault.configure_approle(client,
name=approle_name,
cidr=cidr,
policies=[policy_name])
if new_role or refresh_secrets:
wrapped_secret = vault.generate_role_secret_id(client,
name=approle_name,
cidr=cidr)
secrets.set_role_id(unit=unit,
role_id=approle_id,
token=wrapped_secret)
clear_flag('endpoint.secrets.new-request')
clear_flag('secrets.refresh')
def _assess_status():
"""Assess status of relations and services for local unit"""
if is_flag_set('snap.channel.invalid'):
status_set(
'blocked', 'Invalid snap channel '
'configured: {}'.format(config('channel')))
return
if is_flag_set('config.dns_vip.invalid'):
status_set('blocked', 'vip and dns-ha-access-record configured')
return
if unitdata.kv().get('charm.vault.series-upgrading'):
status_set(
"blocked", "Ready for do-release-upgrade and reboot. "
"Set complete when finished.")
return
if is_flag_set('failed.to.start'):
status_set("blocked",
"Vault failed to start; check journalctl -u vault")
return
_missing_interfaces = []
_incomplete_interfaces = []
_assess_interface_groups(REQUIRED_INTERFACES,
optional=False,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
_assess_interface_groups(OPTIONAL_INTERFACES,
optional=True,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
if _missing_interfaces or _incomplete_interfaces:
state = 'blocked' if _missing_interfaces else 'waiting'
status_set(state,
', '.join(_missing_interfaces + _incomplete_interfaces))
return
health = None
if service_running('vault'):
try:
health = vault.get_vault_health()
except Exception:
log(traceback.format_exc(), level=ERROR)
status_set('blocked', 'Vault health check failed')
return
else:
status_set('blocked', 'Vault service not running')
return
if health.get('version'):
application_version_set(health.get('version'))
else:
application_version_set('Unknown')
status_set('blocked', 'Unknown vault version')
return
if not health['initialized']:
status_set('blocked', 'Vault needs to be initialized')
return
if health['sealed']:
status_set('blocked', 'Unit is sealed')
return
if not leader_get(vault.CHARM_ACCESS_ROLE_ID):
status_set(
'blocked',
'Vault charm not yet authorized: run authorize-charm action.')
return
if not client_approle_authorized():
status_set('blocked', 'Vault cannot authorize approle')
return
mlock_disabled = is_container() or config('disable-mlock')
vault_installed_version = snap.get_installed_version('vault')
vault_running_version = health.get('version')
if vault_installed_version != vault_running_version:
status_set(
'active',
'New version of vault installed, manual intervention required '
'to restart the service.')
return
status_set(
'active', 'Unit is ready '
'(active: {}, mlock: {})'.format(
str(not health['standby']).lower(),
'disabled' if mlock_disabled else 'enabled'))
def message():
"""Set a message to notify the user that this charm is ready."""
if is_flag_set("client.available"):
status.active("Certificate Authority connected.")
else:
status.active("Certificate Authority ready.")
