def config_changed():
# if we are paused, delay doing any config changed hooks.
# It is forced on the resume.
if is_unit_paused_set():
log("Unit is pause or upgrading. Skipping config_changed", "WARN")
return
# If neutron is ready to be queried then check for incompatability between
# existing neutron objects and charm settings
if neutron_ready():
if l3ha_router_present() and not get_l3ha():
e = ('Cannot disable Router HA while ha enabled routers exist.'
' Please remove any ha routers')
status_set('blocked', e)
raise Exception(e)
if dvr_router_present() and not get_dvr():
e = ('Cannot disable dvr while dvr enabled routers exist. Please'
' remove any distributed routers')
log(e, level=ERROR)
status_set('blocked', e)
raise Exception(e)
if config('prefer-ipv6'):
status_set('maintenance', 'configuring ipv6')
setup_ipv6()
sync_db_with_multi_ipv6_addresses(config('database'),
config('database-user'))
global CONFIGS
if not config('action-managed-upgrade'):
if openstack_upgrade_available('neutron-common'):
status_set('maintenance', 'Running openstack upgrade')
do_openstack_upgrade(CONFIGS)
additional_install_locations(
config('neutron-plugin'),
config('openstack-origin')
)
status_set('maintenance', 'Installing apt packages')
apt_install(filter_installed_packages(
determine_packages(config('openstack-origin'))),
fatal=True)
packages_removed = remove_old_packages()
configure_https()
update_nrpe_config()
CONFIGS.write_all()
if packages_removed and not is_unit_paused_set():
log("Package purge detected, restarting services", "INFO")
for s in services():
service_restart(s)
for r_id in relation_ids('neutron-api'):
neutron_api_relation_joined(rid=r_id)
for r_id in relation_ids('neutron-plugin-api'):
neutron_plugin_api_relation_joined(rid=r_id)
for r_id in relation_ids('amqp'):
amqp_joined(relation_id=r_id)
for r_id in relation_ids('identity-service'):
identity_joined(rid=r_id)
for r_id in relation_ids('ha'):
ha_joined(relation_id=r_id)
[cluster_joined(rid) for rid in relation_ids('cluster')]
def download_from_upstream():
if not config('fallback_url') or not config('fallback_sum'):
status_set('blocked', 'Missing configuration: ')
return None
client = ArchiveUrlFetchHandler()
return client.download_and_validate(config('fallback_url'),
config('fallback_sum'))
def install():
"""Install REST2JFed"""
try:
# update needed because of weird error
hookenv.log("Installing dependencies")
subprocess.check_output(['apt-get', 'update'])
subprocess.check_output(['pip2', 'install', 'Jinja2', 'Flask', 'pyyaml', 'click', 'python-dateutil'])
except subprocess.CalledProcessError as exception:
hookenv.log(exception.output)
exit(1)
hookenv.log("Extracting and moving required files and folders")
mergecopytree(charm_dir() + '/files/jfedS4', "/opt/jfedS4")
mergecopytree(charm_dir() + '/files/rest2jfed', "/opt/rest2jfed")
hookenv.log("Generating upstart file")
with open(charm_dir()+'/templates/upstart.conf', 'r') as upstart_t_file:
upstart_template = upstart_t_file.read()
with open('/etc/init/rest2jfed.conf', 'w') as upstart_file:
upstart_file = upstart_file.write(upstart_template)
hookenv.log("Starting rest2jfed service")
try:
subprocess.check_output(['service', 'rest2jfed', 'start'])
except subprocess.CalledProcessError as exception:
hookenv.log(exception.output)
exit(1)
open_port(5000)
status_set('active', 'Ready')
set_state('rest2jfed.installed')
def _validate_json(self, data):
try:
return json.loads(data)
except ValueError as e:
msg = "JSON Error: {}".format(e.message)
status_set('blocked', msg)
raise ErrorException(msg)
def install_dockerbeat():
''' Installs dockerbeat from resources, with a fallback option
to try to fetch over the network, for 1.25.5 hosts'''
try:
bin_path = resource_get('dockerbeat')
except NotImplementedError:
# Attempt to fetch and install from configured uri with validation
bin_path = download_from_upstream()
full_beat_path = '/usr/local/bin/dockerbeat'
if not bin_path:
status_set('blocked', 'Missing dockerbeat binary')
return
install(bin_path, full_beat_path)
os.chmod(full_beat_path, 0o755)
codename = lsb_release()['DISTRIB_CODENAME']
# render the apropriate init systems configuration
if codename == 'trusty':
render('upstart', '/etc/init/dockerbeat.conf', {})
else:
render('systemd', '/etc/systemd/system/dockerbeat.service', {})
set_state('dockerbeat.installed')
def request_integration():
hookenv.status_set('maintenance', 'requesting cloud integration')
kube_control = endpoint_from_flag('kube-control.cluster_tag.available')
cluster_tag = kube_control.get_cluster_tag()
if is_state('endpoint.aws.joined'):
cloud = endpoint_from_flag('endpoint.aws.joined')
cloud.tag_instance({
'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned',
})
cloud.tag_instance_security_group({
'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned',
})
cloud.tag_instance_subnet({
'kubernetes.io/cluster/{}'.format(cluster_tag): 'owned',
})
cloud.enable_object_storage_management(['kubernetes-*'])
elif is_state('endpoint.gcp.joined'):
cloud = endpoint_from_flag('endpoint.gcp.joined')
cloud.label_instance({
'k8s-io-cluster-name': cluster_tag,
})
cloud.enable_object_storage_management()
cloud.enable_instance_inspection()
cloud.enable_dns_management()
set_state('kubernetes-worker.cloud-request-sent')
hookenv.status_set('waiting', 'waiting for cloud integration')
def enable_gpu():
"""Enable GPU usage on this node.
"""
if get_version('kubelet') < (1, 9):
hookenv.status_set(
'active',
'Upgrade to snap channel >= 1.9/stable to enable GPU suppport.'
)
return
hookenv.log('Enabling gpu mode')
try:
# Not sure why this is necessary, but if you don't run this, k8s will
# think that the node has 0 gpus (as shown by the output of
# `kubectl get nodes -o yaml`
check_call(['nvidia-smi'])
except CalledProcessError as cpe:
hookenv.log('Unable to communicate with the NVIDIA driver.')
hookenv.log(cpe)
return
set_label('gpu', 'true')
set_label('cuda', 'true')
set_state('kubernetes-worker.gpu.enabled')
set_state('kubernetes-worker.restart-needed')
def create_tenant():
status_set("maintenance", "creating tenant")
render(source="tenant.yaml", target="/tmp/tenant.yaml", owner="openvim", perms=0o664, context={})
cmd = 'openvim tenant-create /tmp/tenant.yaml'
tenant_uuid = sh_as_openvim(cmd).split()[0]
tenant_uuid = str(tenant_uuid, 'utf-8')
return tenant_uuid
def create_image():
status_set("maintenance", "creating image")
render(source="image.yaml", target="/tmp/image.yaml", owner="openvim", perms=0o664, context={})
cmd = 'openvim image-create /tmp/image.yaml'
image_uuid = sh_as_openvim(cmd).split()[0]
image_uuid = str(image_uuid, 'utf-8')
return image_uuid
def create_flavor():
status_set("maintenance", "creating flavor")
render(source="flavor.yaml", target="/tmp/flavor.yaml", owner="openvim", perms=0o664, context={})
cmd = 'openvim flavor-create /tmp/flavor.yaml'
flavor_uuid = sh_as_openvim(cmd).split()[0]
flavor_uuid = str(flavor_uuid, 'utf-8')
return flavor_uuid
def create_virbr_network():
status_set("maintenance", "creating virbr0 network")
render(source="net-virbr0.yaml", target="/tmp/net-virbr0.yaml", owner="openvim", perms=0o664, context={})
cmd = 'openvim net-create /tmp/net-virbr0.yaml'
net_virbr0_uuid = sh_as_openvim(cmd).split()[0]
net_virbr0_uuid = str(net_virbr0_uuid, 'utf-8')
return net_virbr0_uuid
def configure_with_remote_db(db):
hookenv.status_set('maintenance', 'configuring external database')
hive = Hive()
hive.configure_remote_db(db)
hive.restart()
set_state('hive.db.configured')
report_status()
def send_nm_all_info(nodemanager):
"""Send nodemanagers all mapred-slave relation data.
At this point, the resourcemanager is ready to serve nodemanagers. Send all
mapred-slave relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
nodemanager.send_resourcemanagers([rm_host])
nodemanager.send_spec(bigtop.spec())
nodemanager.send_ports(rm_ipc, jh_http, jh_ipc)
# hosts_map and ssh_key are required by the mapred-slave interface to signify
# RM's readiness. Send them, even though they are not utilized by bigtop.
# NB: update KV hosts with all nodemanagers prior to sending the hosts_map
# because mapred-slave gates readiness on a NM's presence in the hosts_map.
utils.update_kv_hosts(nodemanager.hosts_map())
nodemanager.send_hosts_map(utils.get_kv_hosts())
nodemanager.send_ssh_key('invalid')
# update status with slave count and report ready for hdfs
num_slaves = len(nodemanager.nodes())
hookenv.status_set('active', 'ready ({count} nodemanager{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-resourcemanager.ready')
def install_nodemanager(namenode, resourcemanager):
"""Install if we have FQDNs.
We only need the master FQDNs to perform the nodemanager install, so poll
for master host data from the appropriate relation. This allows us to
install asap, even if '<master>.ready' is not set.
"""
namenodes = namenode.namenodes()
resourcemanagers = resourcemanager.resourcemanagers()
masters = namenodes + resourcemanagers
if namenodes and resourcemanagers and data_changed('nm.masters', masters):
installed = is_state('apache-bigtop-nodemanager.installed')
action = 'installing' if not installed else 'configuring'
hookenv.status_set('maintenance', '%s nodemanager' % action)
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': namenodes[0],
'resourcemanager': resourcemanagers[0],
},
roles=[
'nodemanager',
'mapred-app',
],
)
bigtop.queue_puppet()
set_state('apache-bigtop-nodemanager.pending')
def enable_gpu():
"""Enable GPU usage on this node.
"""
config = hookenv.config()
if config['allow-privileged'] == "false":
hookenv.status_set(
'active',
'GPUs available. Set allow-privileged="auto" to enable.'
)
return
hookenv.log('Enabling gpu mode')
try:
# Not sure why this is necessary, but if you don't run this, k8s will
# think that the node has 0 gpus (as shown by the output of
# `kubectl get nodes -o yaml`
check_call(['nvidia-smi'])
except CalledProcessError as cpe:
hookenv.log('Unable to communicate with the NVIDIA driver.')
hookenv.log(cpe)
return
# Apply node labels
_apply_node_label('gpu=true', overwrite=True)
_apply_node_label('cuda=true', overwrite=True)
set_state('kubernetes-worker.gpu.enabled')
set_state('kubernetes-worker.restart-needed')
def leader_init_db_if_ready(skip_acl_check=False, skip_cells_restarts=False,
db_rid=None, unit=None):
"""Initialise db if leader and db not yet intialised.
NOTE: must be called from database context.
"""
if not is_elected_leader(CLUSTER_RES):
log("Not leader - skipping db init", level=DEBUG)
return
if is_db_initialised():
log("Database already initialised - skipping db init", level=DEBUG)
return
# Bugs 1353135 & 1187508. Dbs can appear to be ready before the units
# acl entry has been added. So, if the db supports passing a list of
# permitted units then check if we're in the list.
allowed_units = relation_get('nova_allowed_units', rid=db_rid, unit=unit)
if skip_acl_check or (allowed_units and local_unit() in
allowed_units.split()):
status_set('maintenance', 'Running nova db migration')
migrate_nova_database()
log('Triggering remote cloud-compute restarts.')
[compute_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cloud-compute')]
log('Triggering remote neutron-network-service restarts.')
[quantum_joined(rid=rid, remote_restart=True)
for rid in relation_ids('quantum-network-service')]
if not skip_cells_restarts:
log('Triggering remote cell restarts.')
[nova_cell_relation_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cell')]
else:
log('allowed_units either not presented, or local unit '
'not in acl list: %s' % repr(allowed_units))
def puppet_active(self):
if config['auto-start']:
hookenv.status_set('active',
'Puppet-agent running')
else:
hookenv.status_set('active',
'Puppet-agent installed, but not running')
def bundle(cmd):
""" Runs bundle
Usage:
bundle('install')
bundle('exec rails s')
bundle('rake db:create RAILS_ENV=production')
Arguments:
cmd: Command to run can be string or list
Returns:
Will halt on error
"""
sh = shell('which bundler')
if sh.code > 0:
gem('install -N bundler')
hookenv.status_set('maintenance', 'Running Bundler')
os.chdir(ruby_dist_dir())
if not isinstance(cmd, str):
hookenv.log('{} must be a string'.format(cmd), 'error')
sys.exit(1)
shell_cmd = set_proxy("bundle {} -j{}".format(cmd, cpu_count()))
sh = shell(shell_cmd, record_output=False)
if sh.code > 0:
hookenv.status_set("blocked", "Ruby error: {}".format(sh.errors()))
hookenv.log("Ruby error: {}".format(sh.errors()))
sys.exit(1)
def configure_zookeeper(zks):
hookenv.status_set('maintenance', 'Configuring Hue for Zookeeper')
hue = Hue(get_dist_config())
hue.configure_zookeeper(zks.zookeepers())
hue.update_apps()
hue.restart()
set_state('zookeeper.configured')
def bootstrap_pxc():
"""Bootstrap PXC
On systemd systems systemctl bootstrap-pxc mysql does not work.
Run service mysql bootstrap-pxc to bootstrap."""
service('stop', 'mysql')
bootstrapped = service('bootstrap-pxc', 'mysql')
if not bootstrapped:
try:
cmp_os = CompareHostReleases(
lsb_release()['DISTRIB_CODENAME']
)
if cmp_os < 'bionic':
# NOTE(jamespage): execute under systemd-run to ensure
# that the bootstrap-pxc mysqld does
# not end up in the juju unit daemons
# cgroup scope.
cmd = ['systemd-run', '--service-type=forking',
'service', 'mysql', 'bootstrap-pxc']
subprocess.check_call(cmd)
else:
service('start', 'mysql@bootstrap')
except subprocess.CalledProcessError as e:
msg = 'Bootstrap PXC failed'
error_msg = '{}: {}'.format(msg, e)
status_set('blocked', msg)
log(error_msg, ERROR)
raise Exception(error_msg)
if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) < 'bionic':
# To make systemd aware mysql is running after a bootstrap
service('start', 'mysql')
log("Bootstrap PXC Succeeded", DEBUG)
def install_load_balancer(apiserver, tls):
''' Create the default vhost template for load balancing '''
# Get the tls paths from the layer data.
layer_options = layer.options('tls-client')
server_cert_path = layer_options.get('server_certificate_path')
cert_exists = server_cert_path and os.path.isfile(server_cert_path)
server_key_path = layer_options.get('server_key_path')
key_exists = server_key_path and os.path.isfile(server_key_path)
# Do both the the key and certificate exist?
if cert_exists and key_exists:
# At this point the cert and key exist, and they are owned by root.
chown = ['chown', 'www-data:www-data', server_cert_path]
# Change the owner to www-data so the nginx process can read the cert.
subprocess.call(chown)
chown = ['chown', 'www-data:www-data', server_key_path]
# Change the owner to www-data so the nginx process can read the key.
subprocess.call(chown)
hookenv.open_port(hookenv.config('port'))
services = apiserver.services()
nginx.configure_site(
'apilb',
'apilb.conf',
server_name='_',
services=services,
port=hookenv.config('port'),
server_certificate=server_cert_path,
server_key=server_key_path,
)
hookenv.status_set('active', 'Loadbalancer ready.')
def stop_spark():
from charms.spark import Spark # in lib/charms; not available until after bootstrap
hookenv.status_set('maintenance', 'Stopping Apache Spark')
spark = Spark()
spark.close_ports()
spark.stop()
def reinstall_spark():
spark_master_host = leadership.leader_get('master-fqdn')
peers = []
zks = []
if is_state('zookeeper.ready'):
# if ZK is availuable we are in HA. We do not want reconfigurations if a leader fails
# HA takes care of this
spark_master_host = ''
zk = RelationBase.from_state('zookeeper.ready')
zks = zk.zookeepers()
# We need reconfigure Spark when in HA and peers change ignore otherwise
peers = get_spark_peers()
deployment_matrix = {
'spark_master': spark_master_host,
'yarn_ready': is_state('hadoop.yarn.ready'),
'hdfs_ready': is_state('hadoop.hdfs.ready'),
'zookeepers': zks,
'peers': peers,
}
if not data_changed('deployment_matrix', deployment_matrix):
return
hookenv.status_set('maintenance', 'configuring spark')
hadoop = (RelationBase.from_state('hadoop.yarn.ready') or
RelationBase.from_state('hadoop.hdfs.ready'))
if install_spark(hadoop, zks):
if is_state('hadoop.yarn.ready'):
set_deployment_mode_state('spark.yarn.installed')
else:
set_deployment_mode_state('spark.standalone.installed')
report_status()
def install_presentation():
""" Install presentation
"""
opts = layer.options('git-deploy')
# Clone repo
hookenv.status_set('maintenance',
'Installing and building the presentation.')
# Build and install
with chdir(opts.get('target')):
with open('requirements.txt', 'r') as f:
for i in list(map(lambda b: b.strip('\n'), f.readlines())):
pip_install(i)
sphinx_build_cmd = 'sphinx-build -b html source %s' % opts.get('target')
subprocess.call(sphinx_build_cmd.split(), shell=False)
present_chown_cmd = 'chown -R www-data:www-data %s' % opts.get('target')
subprocess.call(present_chown_cmd.split(), shell=False)
# Configure nginx vhost
configure_site('present', 'present.vhost', app_path=opts.get('target'))
# Open presentation front-end port
hookenv.open_port(config['port'])
# Set status
hookenv.status_set('active',
'Presentation is active on port %s' % config['port'])
# Set state
set_state('presentation.available')
def assess_status(self):
"""Assess the status of the unit and set the status and a useful
message as appropriate.
The 3 checks are:
1. Check if the unit has been paused (using
os_utils.is_unit_paused_set().
2. Check if the interfaces are all present (using the states that are
set by each interface as it comes 'live'.
3. Do a custom_assess_status_check() check.
4. Check that services that should be running are running.
Each sub-function determins what checks are taking place.
If custom assess_status() functionality is required then the derived
class should override any of the 4 check functions to alter the
behaviour as required.
Note that if ports are NOT to be checked, then the derived class should
override :meth:`ports_to_check()` and return an empty list.
SIDE EFFECT: this function calls status_set(state, message) to set the
workload status in juju.
"""
for f in [self.check_if_paused,
self.check_interfaces,
self.custom_assess_status_check,
self.check_services_running]:
state, message = f()
if state is not None:
hookenv.status_set(state, message)
return
# No state was particularly set, so assume the unit is active
hookenv.status_set('active', 'Unit is ready')
def get_keys_from_leader(keys, overwrite_local=False):
"""
Gets the broadcasted keys from the leader and stores them in
the corresponding files.
Args:
keys: list of keys. Keys are actually files on the FS.
Returns: True if all key were fetched, False if not.
"""
# This races with other codepaths, and seems to require being created first
# This block may be extracted later, but for now seems to work as intended
os.makedirs('/root/cdk', exist_ok=True)
for k in keys:
# If the path does not exist, assume we need it
if not os.path.exists(k) or overwrite_local:
# Fetch data from leadership broadcast
contents = charms.leadership.leader_get(k)
# Default to logging the warning and wait for leader data to be set
if contents is None:
msg = "Waiting on leaders crypto keys."
hookenv.status_set('waiting', msg)
hookenv.log('Missing content for file {}'.format(k))
return False
# Write out the file and move on to the next item
with open(k, 'w+') as fp:
fp.write(contents)
fp.write('\n')
return True
def restart_scheduler():
prev_state, prev_msg = hookenv.status_get()
hookenv.status_set('maintenance', 'Restarting kube-scheduler')
host.service_restart('snap.kube-scheduler.daemon')
hookenv.status_set(prev_state, prev_msg)
remove_state('kube-scheduler.do-restart')
set_state('kube-scheduler.started')
def stop_cassandra():
if is_cassandra_running():
hookenv.log('Shutting down Cassandra')
host.service_stop(get_cassandra_service())
if is_cassandra_running():
hookenv.status_set('blocked', 'Cassandra failed to shut down')
raise SystemExit(0)
def run_che():
status_set('maintenance', 'Installing Eclipse Che')
# Start and stop Che so che's config is generated
start_che()
stop_che()
# Add Juju stuff to Che config
json_add_object_to_array(
"{}/templates/stack-juju-charm.json".format(charm_dir()),
"/home/ubuntu/instance/data/stacks/stacks.json"
)
copyfile(
"{}/templates/type-juju.svg".format(charm_dir()),
"/home/ubuntu/instance/data/stacks/images/type-juju.svg"
)
json_add_object_to_array(
"{}/templates/project-template-charms.json".format(charm_dir()),
"/home/ubuntu/instance/data/templates/samples.json"
)
json_add_object_to_array(
"{}/templates/project-template-interface.json".format(charm_dir()),
"/home/ubuntu/instance/data/templates/samples.json"
)
json_add_object_to_array(
"{}/templates/project-template-layer.json".format(charm_dir()),
"/home/ubuntu/instance/data/templates/samples.json"
)
# Start Che for real
start_che()
# opened ports are used by `juju expose` so It's important to open all
# ports a user connects to.
open_port('8080', protocol="TCP") # Port to the UI
open_port('32768-65535', protocol="TCP") # Ports to the workspaces
status_set('active', 'Ready (eclipse/che)')
set_state('che.available')
def start(self):
"""
Always start the Spark History Server. Start other services as
required by our execution mode. Open related ports as appropriate.
"""
host.service_start('spark-history-server')
hookenv.open_port(self.dist_config.port('spark-history-ui'))
# Spark master/worker is only started in standalone mode
if hookenv.config()['spark_execution_mode'] == 'standalone':
if host.service_start('spark-master'):
hookenv.log("Spark Master started")
hookenv.open_port(self.dist_config.port('spark-master-ui'))
# If the master started and we have peers, wait 2m for recovery
# before starting the worker. This ensures the worker binds
# to the correct master.
if unitdata.kv().get('sparkpeer.units'):
hookenv.status_set('maintenance',
'waiting for spark master recovery')
hookenv.log("Waiting 2m to ensure spark master is ALIVE")
time.sleep(120)
else:
hookenv.log("Spark Master did not start; this is normal "
"for non-leader units in standalone mode")
# NB: Start the worker even if the master process on this unit
# fails to start. In non-HA mode, spark master only runs on the
# leader. On non-leader units, we still want a worker bound to
# the leader.
if host.service_start('spark-worker'):
hookenv.log("Spark Worker started")
hookenv.open_port(self.dist_config.port('spark-worker-ui'))
else:
hookenv.log("Spark Worker did not start")
def assess_status():
"""Assess status of current unit"""
# check to see if the unit is paused.
application_version_set(get_upstream_version(VERSION_PACKAGE))
if is_unit_paused_set():
status_set('maintenance',
"Paused. Use 'resume' action to resume normal service.")
return
# Check for mon relation
if len(relation_ids('mon')) < 1:
status_set('blocked', 'Missing relation: monitor')
return
# Check for monitors with presented addresses
# Check for bootstrap key presentation
monitors = get_mon_hosts()
if len(monitors) < 1 or not get_conf('osd_bootstrap_key'):
status_set('waiting', 'Incomplete relation: monitor')
return
# Check for vault
if use_vaultlocker():
if not relation_ids('secrets-storage'):
status_set('blocked', 'Missing relation: vault')
return
if not vaultlocker.vault_relation_complete():
status_set('waiting', 'Incomplete relation: vault')
return
# Check for OSD device creation parity i.e. at least some devices
# must have been presented and used for this charm to be operational
(prev_status, prev_message) = status_get()
running_osds = ceph.get_running_osds()
if not prev_message.startswith('Non-pristine'):
if not running_osds:
status_set(
'blocked',
'No block devices detected using current configuration')
else:
status_set('active',
'Unit is ready ({} OSD)'.format(len(running_osds)))
def upgrade_needed_status():
msg = 'Needs manual upgrade, run the upgrade action'
hookenv.status_set('blocked', msg)
def zookeeper_wait(zookeeper):
hookenv.status_set('waiting', 'Waiting for Zookeeper to become available')
def apache_started():
status_set('active', 'Ready')
def mysql_connected_but_waiting(mysql):
status_set('waiting', 'Waiting for mysql service')
def no_mysql_relation():
status_set('waiting', 'Waiting for mysql relation')
def reinstall_spark():
"""
This is tricky. We want to fire on config or leadership changes, or when
hadoop, sparkpeers, or zookeepers come and go. In the future this should
fire when Cassandra or any other storage comes or goes. We always fire
this method (or rather, when bigtop is ready and juju has elected a
master). We then build a deployment-matrix and (re)install as things
change.
"""
spark_master_host = leadership.leader_get('master-fqdn')
if not spark_master_host:
hookenv.status_set('maintenance', 'juju leader not elected yet')
return
mode = hookenv.config()['spark_execution_mode']
peers = None
zks = None
# If mode is standalone and ZK is ready, we are in HA. Do not consider
# the master_host from juju leadership in our matrix. ZK handles this.
if (mode == 'standalone' and is_state('zookeeper.ready')):
spark_master_host = ''
zk = RelationBase.from_state('zookeeper.ready')
zks = zk.zookeepers()
# peers are only used to set our MASTER_URL in standalone HA mode
peers = get_spark_peers()
deployment_matrix = {
'spark_master': spark_master_host,
'yarn_ready': is_state('hadoop.yarn.ready'),
'hdfs_ready': is_state('hadoop.hdfs.ready'),
'zookeepers': zks,
'peers': peers,
}
# If neither config nor our matrix is changing, there is nothing to do.
if not (is_state('config.changed')
or data_changed('deployment_matrix', deployment_matrix)):
report_status()
return
# (Re)install based on our execution mode
hookenv.status_set('maintenance',
'configuring spark in {} mode'.format(mode))
hookenv.log("Configuring spark with deployment matrix: {}".format(
deployment_matrix))
if mode.startswith('yarn') and is_state('hadoop.yarn.ready'):
install_spark_yarn()
elif mode.startswith('local') or mode == 'standalone':
install_spark_standalone(zks, peers)
else:
# Something's wrong (probably requested yarn without yarn.ready).
remove_state('spark.started')
report_status()
return
# restart services to pick up possible config changes
spark = Spark()
spark.stop()
spark.start()
set_state('spark.started')
report_status()
def install_platform(self):
hookenv.status_set('maintenance', 'Installing platform')
package_file = self.download_platform()
dpkg_install_platform_deb = 'dpkg -i %s' % package_file
check_call(dpkg_install_platform_deb.split(), shell=False)
hookenv.status_set('active', 'Platform installed')
def open_port():
hookenv.status_set('maintenance', 'Opening port')
hookenv.open_port(8081)
set_flag('candid.port_opened')
def uninstall_platform(self):
hookenv.status_set('maintenance', 'Uninstalling platform')
dpkg_uninstall_platform_deb = 'dpkg --purage hpccsystems-platform'
check_call(dpkg_uninstall_platform_deb.split(), shell=False)
def update_status():
try:
hookenv.application_version_set(candid.get_version())
hookenv.status_set('active', '')
except Exception as e:
hookenv.log("cannot get version: {}".format(e), level="warning")
def compute_changed(rid=None, unit=None):
for r_id in relation_ids('nova-api'):
nova_api_relation_joined(rid=r_id)
rel_settings = relation_get(rid=rid, unit=unit)
if not rel_settings.get('region', None) == config('region'):
relation_set(relation_id=rid, region=config('region'))
if 'migration_auth_type' not in rel_settings:
return
if rel_settings['migration_auth_type'] == 'ssh':
status_set('maintenance', 'configuring live migration')
key = rel_settings.get('ssh_public_key')
if not key:
log('SSH migration set but peer did not publish key.')
return
ssh_compute_add(key, rid=rid, unit=unit)
index = 0
for line in ssh_known_hosts_lines(unit=unit):
relation_set(
relation_id=rid,
relation_settings={
'known_hosts_{}'.format(index): line})
index += 1
relation_set(relation_id=rid, known_hosts_max_index=index)
index = 0
for line in ssh_authorized_keys_lines(unit=unit):
relation_set(
relation_id=rid,
relation_settings={
'authorized_keys_{}'.format(index): line})
index += 1
relation_set(relation_id=rid, authorized_keys_max_index=index)
if 'nova_ssh_public_key' not in rel_settings:
return
if rel_settings['nova_ssh_public_key']:
ssh_compute_add(rel_settings['nova_ssh_public_key'],
rid=rid, unit=unit, user='******')
index = 0
for line in ssh_known_hosts_lines(unit=unit, user='******'):
relation_set(
relation_id=rid,
relation_settings={
'{}_known_hosts_{}'.format(
'nova',
index): line})
index += 1
relation_set(
relation_id=rid,
relation_settings={
'{}_known_hosts_max_index'.format('nova'): index})
index = 0
for line in ssh_authorized_keys_lines(unit=unit, user='******'):
relation_set(
relation_id=rid,
relation_settings={
'{}_authorized_keys_{}'.format(
'nova',
index): line})
index += 1
relation_set(
relation_id=rid,
relation_settings={
'{}_authorized_keys_max_index'.format('nova'): index})
def check_config():
mode = 'mapreduce' if is_state('hadoop.ready') else 'local'
if data_changed('pig.mode', mode):
Pig().update_config(mode)
hookenv.status_set('active', 'ready (%s)' % mode)
def create_keypair():
hookenv.status_set('maintenance', 'Generating default keypair')
key = candid.generate_keypair()
leadership.leader_set({"private-key": key["private"]})
leadership.leader_set({"public-key": key["public"]})
def gitlab_active():
status_set('active', '')
def set_os_workload_status(configs, required_interfaces, charm_func=None):
"""
Set workload status based on complete contexts.
status-set missing or incomplete contexts
and juju-log details of missing required data.
charm_func is a charm specific function to run checking
for charm specific requirements such as a VIP setting.
"""
incomplete_rel_data = incomplete_relation_data(configs, required_interfaces)
state = 'active'
missing_relations = []
incomplete_relations = []
message = None
charm_state = None
charm_message = None
for generic_interface in incomplete_rel_data.keys():
related_interface = None
missing_data = {}
# Related or not?
for interface in incomplete_rel_data[generic_interface]:
if incomplete_rel_data[generic_interface][interface].get('related'):
related_interface = interface
missing_data = incomplete_rel_data[generic_interface][interface].get('missing_data')
# No relation ID for the generic_interface
if not related_interface:
juju_log("{} relation is missing and must be related for "
"functionality. ".format(generic_interface), 'WARN')
state = 'blocked'
if generic_interface not in missing_relations:
missing_relations.append(generic_interface)
else:
# Relation ID exists but no related unit
if not missing_data:
# Edge case relation ID exists but departing
if ('departed' in hook_name() or 'broken' in hook_name()) \
and related_interface in hook_name():
state = 'blocked'
if generic_interface not in missing_relations:
missing_relations.append(generic_interface)
juju_log("{} relation's interface, {}, "
"relationship is departed or broken "
"and is required for functionality."
"".format(generic_interface, related_interface), "WARN")
# Normal case relation ID exists but no related unit
# (joining)
else:
juju_log("{} relations's interface, {}, is related but has "
"no units in the relation."
"".format(generic_interface, related_interface), "INFO")
# Related unit exists and data missing on the relation
else:
juju_log("{} relation's interface, {}, is related awaiting "
"the following data from the relationship: {}. "
"".format(generic_interface, related_interface,
", ".join(missing_data)), "INFO")
if state != 'blocked':
state = 'waiting'
if generic_interface not in incomplete_relations \
and generic_interface not in missing_relations:
incomplete_relations.append(generic_interface)
if missing_relations:
message = "Missing relations: {}".format(", ".join(missing_relations))
if incomplete_relations:
message += "; incomplete relations: {}" \
"".format(", ".join(incomplete_relations))
state = 'blocked'
elif incomplete_relations:
message = "Incomplete relations: {}" \
"".format(", ".join(incomplete_relations))
state = 'waiting'
# Run charm specific checks
if charm_func:
charm_state, charm_message = charm_func(configs)
if charm_state != 'active' and charm_state != 'unknown':
state = workload_state_compare(state, charm_state)
if message:
charm_message = charm_message.replace("Incomplete relations: ",
"")
message = "{}, {}".format(message, charm_message)
else:
message = charm_message
# Set to active if all requirements have been met
if state == 'active':
message = "Unit is ready"
juju_log(message, "INFO")
status_set(state, message)
def leader_elected():
status_set("maintenance", "{} is the elected leader".format(local_unit()))
def install_pig():
hookenv.status_set('maintenance', 'installing pig')
pig = Pig()
pig.install_pig()
pig.initial_pig_config()
set_state('pig.installed')
def check_resources_for_upgrade_needed():
hookenv.status_set('maintenance', 'Checking resources')
resources = ['kubectl', 'kubelet', 'kube-proxy']
paths = [hookenv.resource_get(resource) for resource in resources]
if any_file_changed(paths):
set_upgrade_needed()
def gitlab_blocked():
status_set('blocked', 'Waiting for database')
def restart_controller_manager():
prev_state, prev_msg = hookenv.status_get()
hookenv.status_set('maintenance', 'Restarting kube-controller-manager')
host.service_restart('snap.kube-controller-manager.daemon')
hookenv.status_set(prev_state, prev_msg)
def simple_server_start():
hookenv.open_port(8080)
hookenv.status_set('active', 'Ready')
def prepare_disks_and_activate():
# NOTE: vault/vaultlocker preflight check
vault_kv = vaultlocker.VaultKVContext(vaultlocker.VAULTLOCKER_BACKEND)
context = vault_kv()
if use_vaultlocker() and not vault_kv.complete:
log('Deferring OSD preparation as vault not ready',
level=DEBUG)
return
elif use_vaultlocker() and vault_kv.complete:
log('Vault ready, writing vaultlocker configuration',
level=DEBUG)
vaultlocker.write_vaultlocker_conf(context)
osd_journal = get_journal_devices()
if not osd_journal.isdisjoint(set(get_devices())):
raise ValueError('`osd-journal` and `osd-devices` options must not'
'overlap.')
log("got journal devs: {}".format(osd_journal), level=DEBUG)
# pre-flight check of eligible device pristinity
devices = get_devices()
# if a device has been previously touched we need to consider it as
# non-pristine. If it needs to be re-processed it has to be zapped
# via the respective action which also clears the unitdata entry.
db = kv()
touched_devices = db.get('osd-devices', [])
devices = [dev for dev in devices if dev not in touched_devices]
log('Skipping osd devices previously processed by this unit: {}'
.format(touched_devices))
# filter osd-devices that are file system paths
devices = [dev for dev in devices if dev.startswith('/dev')]
# filter osd-devices that does not exist on this unit
devices = [dev for dev in devices if os.path.exists(dev)]
# filter osd-devices that are already mounted
devices = [dev for dev in devices if not is_device_mounted(dev)]
# filter osd-devices that are active bluestore devices
devices = [dev for dev in devices
if not ceph.is_active_bluestore_device(dev)]
log('Checking for pristine devices: "{}"'.format(devices), level=DEBUG)
if not all(ceph.is_pristine_disk(dev) for dev in devices):
status_set('blocked',
'Non-pristine devices detected, consult '
'`list-disks`, `zap-disk` and `blacklist-*` actions.')
return
if is_osd_bootstrap_ready():
log('ceph bootstrapped, rescanning disks')
emit_cephconf()
bluestore = use_bluestore()
ceph.udevadm_settle()
for dev in get_devices():
ceph.osdize(dev, config('osd-format'),
osd_journal,
config('ignore-device-errors'),
config('osd-encrypt'),
bluestore,
config('osd-encrypt-keymanager'))
# Make it fast!
if config('autotune'):
ceph.tune_dev(dev)
ceph.start_osds(get_devices())
# Notify MON cluster as to how many OSD's this unit bootstrapped
# into the cluster
for r_id in relation_ids('mon'):
relation_set(
relation_id=r_id,
relation_settings={
'bootstrapped-osds': len(db.get('osd-devices', [])),
'ceph_release': ceph.resolve_ceph_version(
hookenv.config('source') or 'distro'
)
}
)
def restart_scheduler():
prev_state, prev_msg = hookenv.status_get()
hookenv.status_set('maintenance', 'Restarting kube-scheduler')
host.service_restart('snap.kube-scheduler.daemon')
hookenv.status_set(prev_state, prev_msg)
def run_sparkler():
status_set('active', 'Sparkler is configured awaiting action')
def assess_status():
"""Assess status of current unit"""
# check to see if the unit is paused.
application_version_set(get_upstream_version(VERSION_PACKAGE))
if is_unit_upgrading_set():
status_set("blocked",
"Ready for do-release-upgrade and reboot. "
"Set complete when finished.")
return
if is_unit_paused_set():
status_set('maintenance',
"Paused. Use 'resume' action to resume normal service.")
return
# Check for mon relation
if len(relation_ids('mon')) < 1:
status_set('blocked', 'Missing relation: monitor')
return
# Check for monitors with presented addresses
# Check for bootstrap key presentation
monitors = get_mon_hosts()
if len(monitors) < 1 or not get_conf('osd_bootstrap_key'):
status_set('waiting', 'Incomplete relation: monitor')
return
# Check for vault
if use_vaultlocker():
if not relation_ids('secrets-storage'):
status_set('blocked', 'Missing relation: vault')
return
if not vaultlocker.vault_relation_complete():
status_set('waiting', 'Incomplete relation: vault')
return
# Check for OSD device creation parity i.e. at least some devices
# must have been presented and used for this charm to be operational
(prev_status, prev_message) = status_get()
running_osds = ceph.get_running_osds()
if not prev_message.startswith('Non-pristine'):
if not running_osds:
status_set('blocked',
'No block devices detected using current configuration')
else:
status_set('active',
'Unit is ready ({} OSD)'.format(len(running_osds)))
else:
pristine = True
osd_journals = get_journal_devices()
for dev in list(set(ceph.unmounted_disks()) - set(osd_journals)):
if (not ceph.is_active_bluestore_device(dev) and
not ceph.is_pristine_disk(dev)):
pristine = False
break
if pristine:
status_set('active',
'Unit is ready ({} OSD)'.format(len(running_osds)))
try:
get_bdev_enable_discard()
except ValueError as ex:
status_set('blocked', str(ex))
def no_java():
status_set('waiting', 'Waiting for Java to become available')
def state_0():
log('rack is up------------------')
status_set('maintenance', 'start: state.0')
time.sleep(TEST_TIMEOUT)
def install_hook():
fetch.add_source(hookenv.config('source'), hookenv.config('key'))
pkgs = fetch.filter_installed_packages(PACKAGES)
if pkgs:
hookenv.status_set('maintenance', 'Installing ganglia packages')
fetch.apt_install(pkgs, fatal=True)
def no_solr():
status_set('waiting', 'Waiting for Solr to become available')
