def client_cert(source, destination, user=None, group=None):
"""
Copy the client certificate to the destination creating directories if
needed and assign ownership if set.
:param string source: The path to look for the certificate, if None
the certificate will be copied from the default location.
:param string destination: The path to save the certificate.
:param string user: The optional name of the user to own the certificate.
:param string group: The optional name of the group to own certificate.
"""
_ensure_directory(destination, user, group)
if not source:
# When source not specified use the default client certificate path.
source = os.path.join(charm_dir(),
'easy-rsa/easyrsa3/pki/issued/client.crt')
# Check for the client certificate.
if os.path.isfile(source):
# Copy the client certificate to the destination.
copy2(source, destination)
else:
# No client certificate file, get the value from unit data.
client_cert_key = 'tls.client.certificate'
# Save the certificate data to the destination.
_save_unitdata(client_cert_key, destination)
chown(destination, user, group)
# Set the destination path for the client certificate path on the unitdata.
unitdata.kv().set('client-cert-path', destination)
def test_check_reverse_dns(self, mock_hookenv, mock_utils,
mock_sub, mock_socket):
'''
Verify that we set the reverse_dns_ok state, and handle errors
correctly.
'''
# Test the case where things succeed.
mock_sub.check_output.return_value = b'domain'
self.bigtop.check_reverse_dns()
self.assertTrue(unitdata.kv().get('reverse_dns_ok'))
# Test the case where we get an exception.
mock_sub.check_output.return_value = b'localdomain'
self.bigtop.check_reverse_dns()
self.assertFalse(unitdata.kv().get('reverse_dns_ok'))
class MockHError(Exception):
pass
def raise_herror(*args, **kwargs):
raise MockHError('test')
mock_socket.herror = MockHError
mock_socket.gethostbyaddr = raise_herror
self.bigtop.check_reverse_dns()
self.assertFalse(unitdata.kv().get('reverse_dns_ok'))
def send_info(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
local_hostname = hookenv.local_unit().replace("/", "-")
hdfs_port = hadoop.dist_config.port("namenode")
webhdfs_port = hadoop.dist_config.port("nn_webapp_http")
utils.update_kv_hosts({node["ip"]: node["host"] for node in datanode.nodes()})
utils.manage_etc_hosts()
datanode.send_spec(hadoop.spec())
datanode.send_namenodes([local_hostname])
datanode.send_ports(hdfs_port, webhdfs_port)
datanode.send_ssh_key(utils.get_ssh_key("hdfs"))
datanode.send_hosts_map(utils.get_kv_hosts())
slaves = [node["host"] for node in datanode.nodes()]
if data_changed("namenode.slaves", slaves):
unitdata.kv().set("namenode.slaves", slaves)
hdfs.register_slaves(slaves)
hookenv.status_set(
"active", "Ready ({count} DataNode{s})".format(count=len(slaves), s="s" if len(slaves) > 1 else "")
)
set_state("namenode.ready")
def install_hadoop(self):
hadoop_version = self.dist_config.hadoop_version
try:
jujuresources.install('hadoop-%s-%s' %
(hadoop_version,
self.cpu_arch),
destination=self.dist_config.path('hadoop'),
skip_top_level=True)
except KeyError:
hookenv.log("Falling back to non-version specific download of hadoop...")
jujuresources.install('hadoop-%s' %
(self.cpu_arch),
destination=self.dist_config.path('hadoop'),
skip_top_level=True)
# Install our lzo compression codec if it's defined in resources.yaml
try:
jujuresources.install('hadoop-lzo-%s' % self.cpu_arch,
destination=self.dist_config.path('hadoop'),
skip_top_level=False)
unitdata.kv().set('hadoop.lzo.installed', True)
except KeyError:
msg = ("The hadoop-lzo-%s resource was not found."
"LZO compression will not be available." % self.cpu_arch)
hookenv.log(msg)
def remove_state(state):
"""Remove / deactivate a state"""
old_states = get_states()
unitdata.kv().unset('reactive.states.%s' % state)
unitdata.kv().set('reactive.dispatch.removed_state', True)
if state in old_states:
StateWatch.change(state)
def initialize_leadership_state():
'''Initialize leadership.* states from the hook environment.
Invoked by hookenv.atstart() so states are available in
@hook decorated handlers.
'''
is_leader = hookenv.is_leader()
if is_leader:
hookenv.log('Initializing Leadership Layer (is leader)')
else:
hookenv.log('Initializing Leadership Layer (is follower)')
reactive.helpers.toggle_state('leadership.is_leader', is_leader)
previous = unitdata.kv().getrange('leadership.settings.', strip=True)
current = hookenv.leader_get()
# Handle deletions.
for key in set(previous.keys()) - set(current.keys()):
current[key] = None
any_changed = False
for key, value in current.items():
reactive.helpers.toggle_state('leadership.changed.{}'.format(key),
value != previous.get(key))
if value != previous.get(key):
any_changed = True
reactive.helpers.toggle_state('leadership.set.{}'.format(key),
value is not None)
reactive.helpers.toggle_state('leadership.changed', any_changed)
unitdata.kv().update(current, prefix='leadership.settings.')
def set_local(self, key=None, value=None, data=None, **kwdata):
"""
Locally store some data associated with this conversation.
Data can be passed in either as a single dict, or as key-word args.
For example, if you need to store the previous value of a remote field
to determine if it has changed, you can use the following::
prev = conversation.get_local('field')
curr = conversation.get_remote('field')
if prev != curr:
handle_change(prev, curr)
conversation.set_local('field', curr)
Note: This uses :mod:`charmhelpers.core.unitdata` and requires that
:meth:`~charmhelpers.core.unitdata.Storage.flush` be called.
:param str key: The name of a field to set.
:param value: A value to set.
:param dict data: A mapping of keys to values.
:param \*\*kwdata: A mapping of keys to values, as keyword arguments.
"""
if data is None:
data = {}
if key is not None:
data[key] = value
data.update(kwdata)
if not data:
return
unitdata.kv().update(data, prefix='%s.%s.' % (self.key, 'local-data'))
def configure_remote_db(self, mysql):
hive_site = self.dist_config.path('hive_conf') / 'hive-site.xml'
jdbc_url = \
"jdbc:mysql://{}:{}/{}?createDatabaseIfNotExist=true".format(
mysql.host(), mysql.port(), mysql.database()
)
with utils.xmlpropmap_edit_in_place(hive_site) as props:
props['javax.jdo.option.ConnectionURL'] = jdbc_url
props['javax.jdo.option.ConnectionUserName'] = mysql.user()
props['javax.jdo.option.ConnectionPassword'] = mysql.password()
props['javax.jdo.option.ConnectionDriverName'] = \
"com.mysql.jdbc.Driver"
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
utils.re_edit_in_place(hive_env, {
r'.*export HIVE_AUX_JARS_PATH *=.*':
('export HIVE_AUX_JARS_PATH='
'/usr/share/java/mysql-connector-java.jar'),
})
# Now that we have db connection info, init our schema (only once)
remote_db = hookenv.remote_service_name()
if not unitdata.kv().get('hive.schema.initialized.%s' % remote_db):
tool_path = "{}/bin/schematool".format(
self.dist_config.path('hive'))
utils.run_as(
'ubuntu', tool_path, '-initSchema', '-dbType', 'mysql')
unitdata.kv().set('hive.schema.initialized.%s' % remote_db, True)
unitdata.kv().flush(True)
def send_info(nodemanager):
hadoop = get_hadoop_base()
yarn = YARN(hadoop)
local_hostname = hookenv.local_unit().replace('/', '-')
port = hadoop.dist_config.port('resourcemanager')
hs_http = hadoop.dist_config.port('jh_webapp_http')
hs_ipc = hadoop.dist_config.port('jobhistory')
utils.update_kv_hosts(nodemanager.hosts_map())
utils.manage_etc_hosts()
nodemanager.send_spec(hadoop.spec())
nodemanager.send_resourcemanagers([local_hostname])
nodemanager.send_ports(port, hs_http, hs_ipc)
nodemanager.send_ssh_key(utils.get_ssh_key('yarn'))
nodemanager.send_hosts_map(utils.get_kv_hosts())
slaves = nodemanager.nodes()
if data_changed('resourcemanager.slaves', slaves):
unitdata.kv().set('resourcemanager.slaves', slaves)
yarn.register_slaves(slaves)
hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format(
count=len(slaves),
s='s' if len(slaves) > 1 else '',
))
set_state('resourcemanager.ready')
def server_key(source, destination, user=None, group=None):
"""
Copy the server key to the destination, creating directories if needed and
assign ownership if set.
:param string source: The directory to look for the key, if None the key
will be copied from default location.
:param string destination: The path to save the key.
:param string user: The optional name of the user to own the key.
:param string group: The optional name of the group to own key.
"""
_ensure_directory(destination, user, group)
if not source:
# Must remove the path characters from the local unit name.
key_name = local_unit().replace('/', '_')
# The location of server key is easy-rsa/easyrsa3/pki/private
source = \
os.path.join(
charm_dir(),
'easy-rsa/easyrsa3/pki/private/{0}.key'.format(key_name))
# Copy the key to the destination.
copy2(source, destination)
chown(destination, user, group)
# Set the destination path for the client key path on the unitdata.
unitdata.kv().set('server-key-path', destination)
def disable_ha(self):
spark_env = self.dist_config.path('spark_conf') / 'spark-env.sh'
utils.re_edit_in_place(spark_env, {
r'.*SPARK_DAEMON_JAVA_OPTS.*': '# SPARK_DAEMON_JAVA_OPTS',
})
unitdata.kv().set('zookeepers.available', False)
unitdata.kv().flush(True)
def _unitdata_cmd(action, key, value):
if action == 'get':
return unitdata.kv().get(key)
elif action == 'set':
unitdata.kv().set(key, value)
unitdata.kv().flush()
return ''
def leader_set(settings=None, **kw):
"""Change leadership settings, per charmhelpers.core.hookenv.leader_set.
The leadership.set.{key} reactive state will be set while the
leadership hook environment setting remains set.
Changed leadership settings will set the leadership.changed.{key}
and leadership.changed states. These states will remain set until
the following hook.
These state changes take effect immediately on the leader, and
in future hooks run on non-leaders. In this way both leaders and
non-leaders can share handlers, waiting on these states.
"""
settings = settings or {}
settings.update(kw)
previous = unitdata.kv().getrange("leadership.settings.", strip=True)
for key, value in settings.items():
if value != previous.get(key):
reactive.set_state("leadership.changed.{}".format(key))
reactive.set_state("leadership.changed")
reactive.helpers.toggle_state("leadership.set.{}".format(key), value is not None)
hookenv.leader_set(settings)
unitdata.kv().update(settings, prefix="leadership.settings.")
def send_info(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
local_hostname = hookenv.local_unit().replace('/', '-')
hdfs_port = hadoop.dist_config.port('namenode')
webhdfs_port = hadoop.dist_config.port('nn_webapp_http')
utils.update_kv_hosts(datanode.hosts_map())
utils.manage_etc_hosts()
datanode.send_spec(hadoop.spec())
datanode.send_namenodes([local_hostname])
datanode.send_ports(hdfs_port, webhdfs_port)
datanode.send_ssh_key(utils.get_ssh_key('hdfs'))
datanode.send_hosts_map(utils.get_kv_hosts())
slaves = datanode.nodes()
if data_changed('namenode.slaves', slaves):
unitdata.kv().set('namenode.slaves', slaves)
hdfs.register_slaves(slaves)
hdfs.refresh_slaves()
hookenv.status_set('active', 'Ready ({count} DataNode{s})'.format(
count=len(slaves),
s='s' if len(slaves) > 1 else '',
))
set_state('namenode.ready')
def ca(source, destination, user=None, group=None):
"""
Copy the Certificate Authority (CA) to the destination, creating parent
directories if needed and assign owner if set. The tls layer installs the
CA on all the peers in /usr/local/share/ca-certificates/.
:param string source: The path to look or the certificate, if None the
CA will be copied from the default location.
:param string destination: The path to save the CA certificate.
:param string user: The optional user name to own the CA certificate.
:param string group: The optional group name to own the CA certificate.
"""
_ensure_directory(destination, user, group)
if not source:
# When source not specified use the default CA path.
source = \
'/usr/local/share/ca-certificates/{0}.crt'.format(service_name())
# Copy the ca certificate to the destination directory.
copy2(source, destination)
chown(destination, user, group)
# Set the destination path for the ca certificate path on the unitdata.
unitdata.kv().set('ca-cert-path', destination)
def join(cls, scope):
"""
Get or create a conversation for the given scope and active hook context.
The current remote unit for the active hook context will be added to
the conversation.
Note: This uses :mod:`charmhelpers.core.unitdata` and requires that
:meth:`~charmhelpers.core.unitdata.Storage.flush` be called.
"""
relation_name = hookenv.relation_type()
relation_id = hookenv.relation_id()
unit = hookenv.remote_unit()
service = hookenv.remote_service_name()
if scope is scopes.UNIT:
scope = unit
namespace = relation_id
elif scope is scopes.SERVICE:
scope = service
namespace = relation_id
else:
namespace = relation_name
key = cls._key(namespace, scope)
data = unitdata.kv().get(key, {'namespace': namespace, 'scope': scope, 'units': []})
conversation = cls.deserialize(data)
conversation.units.add(unit)
unitdata.kv().set(key, cls.serialize(conversation))
return conversation
def server_cert(source, destination, user=None, group=None):
"""
Copy the server certificate to the destination, creating directories if
needed and assign ownership if set.
:param string source: The directory to look for the certificate, if None
the certificate will be copied from unit data.
:param string destination: The path to save the certificate.
:param string user: The optional name of the user to own the certificate.
:param string group: The optional name of the group to own certificate.
"""
_ensure_directory(destination, user, group)
if not source:
# Must remove the path characters from the local unit name.
key_name = local_unit().replace('/', '_')
# The location of server certificate is easy-rsa/easyrsa3/pki/issued
source = \
os.path.join(
charm_dir(),
'easy-rsa/easyrsa3/pki/issued/{0}.crt'.format(key_name))
if os.path.isfile(source):
# Copy the server certificate to the destination.
copy2(source, destination)
else:
# No source server certificate, get the value from unit data.
server_cert_key = 'tls.server.certificate'
# Save the certificate data to the destination directory.
_save_unitdata(server_cert_key, destination)
chown(destination, user, group)
# Set the destination path for the client certificate path on the unitdata.
unitdata.kv().set('server-cert-path', destination)
def clear_config_states():
from charmhelpers.core import hookenv, unitdata
from charms.reactive import remove_state
config = hookenv.config()
remove_state('config.changed')
for opt in config.keys():
remove_state('config.changed.{}'.format(opt))
unitdata.kv().flush()
def set_state(state, value=None):
"""
Set the given state as active, optionally associating with a relation.
"""
old_states = get_states()
unitdata.kv().update({state: value}, prefix='reactive.states.')
if state not in old_states:
StateWatch.change(state)
def invoke(self):
"""
Call the external handler to be invoked.
"""
# flush to ensure external process can see states as they currently
# are, and write states (flush releases lock)
unitdata.kv().flush()
subprocess.check_call([self._filepath, '--invoke', self._test_output], env=os.environ)
def set_cert(key, certificate):
"""Set the certificate on the key value store of the unit, and set
the corresponding state for layers to consume."""
# Set cert on the unitdata key value store so other layers can get it.
unitdata.kv().set(key, certificate)
# Set the final state for the other layers to know when they can
# retrieve the server certificate.
set_state("{0} available".format(key))
def action(self):
"""
Execute the external handler, which should perform its own predicate
and reinvocation checking.
"""
# flush to ensure external process can see states as they currently
# are, and write states (flush releases lock)
unitdata.kv().flush()
subprocess.check_call([self.filepath], env=os.environ)
def trigger_bigtop(self):
'''
Trigger the Bigtop puppet recipe that handles the Zeppelin service.
'''
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
# The zep deb depends on spark-core which unfortunately brings in
# most of hadoop. Include appropriate roles here to ensure these
# packages are configured in the same way as our other Bigtop
# software deployed with puppet.
bigtop.render_site_yaml(
roles=[
'spark-client',
'spark-yarn-slave',
'zeppelin-server',
],
overrides=overrides,
)
# NB: during an upgrade, we configure the site.yaml, but do not
# trigger puppet. The user must do that with the 'reinstall' action.
if unitdata.kv().get('zeppelin.version.repo', False):
hookenv.log("An upgrade is available and the site.yaml has been "
"configured. Run the 'reinstall' action to continue.",
level=hookenv.INFO)
else:
####################################################################
# BUG: BIGTOP-2742
# Default zeppelin init script looks for the literal '$(hostname)'
# string. Symlink it so it exists before the apt install from puppet
# tries to start the service.
import subprocess
host = subprocess.check_output(['hostname']).decode('utf8').strip()
zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host)
utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin')
utils.run_as('root', 'ln', '-sf',
zepp_pid,
'/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid')
####################################################################
bigtop.trigger_puppet()
self.wait_for_api(30)
####################################################################
# BUG: BIGTOP-2742
# Puppet apply will call systemctl daemon-reload, which removes the
# symlink we just created. Now that the bits are on disk, update the
# init script $(hostname) that caused this mess to begin with.
zepp_init_script = '/etc/init.d/zeppelin'
utils.re_edit_in_place(zepp_init_script, {
r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid),
})
utils.run_as('root', 'systemctl', 'daemon-reload')
self.restart()
self.wait_for_api(30)
def init_zkrest(self):
# Zookeeper user needs to compile the rest contrib server.
# So zookeeper needs to:
# 1. Have a home dir for ant cache to exist
# 2. Write to the /usr/lib/zookeeper
chownr(self.dist_config.path('zookeeper'), 'zookeeper', 'zookeeper', chowntopdir=True)
with chdir(self.dist_config.path('zookeeper')):
utils.run_as('zookeeper', 'ant')
unitdata.kv().set('rest.initialised', True)
def update_active_status():
datanode = DataNode()
if datanode.is_ready():
hookenv.status_set('active', 'Ready (%s DataNodes)' % len(datanode.filtered_data()))
unitdata.kv().set('charm.active', True)
elif datanode.connected_units():
hookenv.status_set('waiting', 'Waiting for compute slaves to provide DataNodes')
else:
hookenv.status_set('blocked', 'Waiting for relation to compute slaves')
def format_namenode(self):
if unitdata.kv().get('hdfs.namenode.formatted'):
return
self.stop_namenode()
# Run without prompting; this will fail if the namenode has already
# been formatted -- we do not want to reformat existing data!
self._hdfs('namenode', '-format', '-noninteractive')
unitdata.kv().set('hdfs.namenode.formatted', True)
unitdata.kv().flush(True)
def update_active_status():
nodemanager = NodeManager()
if nodemanager.is_ready():
hookenv.status_set('active', 'Ready (%s NodeManagers)' % len(nodemanager.filtered_data()))
unitdata.kv().set('charm.active', True)
elif nodemanager.connected_units():
hookenv.status_set('waiting', 'Waiting for compute slaves to provide NodeManagers')
else:
hookenv.status_set('blocked', 'Waiting for relation to compute slaves')
def any_changed():
changed = False
for filename in filenames:
old_hash = unitdata.kv().get('reactive.when_file_changed.%s' % filename)
new_hash = host.file_hash(filename, hash_type=kwargs.get('hash_type', 'md5'))
if old_hash != new_hash:
unitdata.kv().set('reactive.when_file_changed.%s' % filename, new_hash)
changed = True # mark as changed, but keep updating hashes
return changed
def install():
'''Install the easy-rsa software that is used by this layer.'''
easyrsa_resource = None
try:
# Try to get the resource from Juju.
easyrsa_resource = resource_get('easyrsa')
except Exception as e:
message = 'An error occurred fetching the easyrsa resource.'
hookenv.log(message)
hookenv.log(e)
hookenv.status_set('blocked', message)
return
if not easyrsa_resource:
hookenv.status_set('blocked', 'The easyrsa resource is missing.')
return
# Get the filesize in bytes.
filesize = os.stat(easyrsa_resource).st_size
# When the filesize is less than 10 KB we do not have a real file.
if filesize < 10240:
hookenv.status_set('blocked', 'The easyrsa resource is not complete.')
return
# Expand the archive in the charm directory creating an EasyRSA directory.
untar = 'tar -xvzf {0} -C {1}'.format(easyrsa_resource, charm_directory)
check_call(split(untar))
version = get_version(easyrsa_resource)
# Save the version in the key/value store of the charm.
unitdata.kv().set('easyrsa-version', version)
if os.path.islink(easyrsa_directory):
check_call(split('rm -v {0}'.format(easyrsa_directory)))
# Link the EasyRSA version directory to a common name.
link = 'ln -v -s {0}/EasyRSA-{1} {2}'.format(charm_directory,
version,
easyrsa_directory)
check_call(split(link))
# The charm pki directory contains backup of pki for upgrades.
charm_pki_directory = os.path.join(charm_directory, 'pki')
if os.path.isdir(charm_pki_directory):
new_pki_directory = os.path.join(easyrsa_directory, 'pki')
# Only copy the directory if the new_pki_directory does not exist.
if not os.path.isdir(new_pki_directory):
# Copy the pki to this new directory.
shutil.copytree(charm_pki_directory, new_pki_directory,
symlinks=True)
# We are done with the old charm pki directory, so delete contents.
shutil.rmtree(charm_pki_directory)
else:
# Create new pki.
with chdir(easyrsa_directory):
check_call(split('./easyrsa --batch init-pki 2>&1'))
set_state('easyrsa.installed')
def install(self):
jujuresources.install(self.resources['hue'],
destination=self.dist_config.path('hue'),
skip_top_level=True)
self.dist_config.add_users()
self.dist_config.add_dirs()
self.dist_config.add_packages()
chownr(self.dist_config.path('hue'), 'hue', 'hadoop')
unitdata.kv().set('hue.installed', True)
def set_meta_generation(gen):
unitdata.kv().set(kvdata.KEY_META_GENERATION, gen)
def __call__(self):
try:
import hvac
except ImportError:
# BUG: #1862085 - if the relation is made to vault, but the
# 'encrypt' option is not made, then the charm errors with an
# import warning. This catches that, logs a warning, and returns
# with an empty context.
hookenv.log(
"VaultKVContext: trying to use hvac pythong module "
"but it's not available. Is secrets-stroage relation "
"made, but encrypt option not set?",
level=hookenv.WARNING)
# return an emptry context on hvac import error
return {}
ctxt = {}
# NOTE(hopem): see https://bugs.launchpad.net/charm-helpers/+bug/1849323
db = unitdata.kv()
# currently known-good secret-id
secret_id = db.get('secret-id')
for relation_id in hookenv.relation_ids(self.interfaces[0]):
for unit in hookenv.related_units(relation_id):
data = hookenv.relation_get(unit=unit, rid=relation_id)
vault_url = data.get('vault_url')
role_id = data.get('{}_role_id'.format(hookenv.local_unit()))
token = data.get('{}_token'.format(hookenv.local_unit()))
if all([vault_url, role_id, token]):
token = json.loads(token)
vault_url = json.loads(vault_url)
# Tokens may change when secret_id's are being
# reissued - if so use token to get new secret_id
token_success = False
try:
secret_id = retrieve_secret_id(url=vault_url,
token=token)
token_success = True
except hvac.exceptions.InvalidRequest:
# Try next
pass
if token_success:
db.set('secret-id', secret_id)
db.flush()
ctxt['vault_url'] = vault_url
ctxt['role_id'] = json.loads(role_id)
ctxt['secret_id'] = secret_id
ctxt['secret_backend'] = self.secret_backend
vault_ca = data.get('vault_ca')
if vault_ca:
ctxt['vault_ca'] = json.loads(vault_ca)
self.complete = True
break
else:
if secret_id:
ctxt['vault_url'] = vault_url
ctxt['role_id'] = json.loads(role_id)
ctxt['secret_id'] = secret_id
ctxt['secret_backend'] = self.secret_backend
vault_ca = data.get('vault_ca')
if vault_ca:
ctxt['vault_ca'] = json.loads(vault_ca)
if self.complete:
break
if ctxt:
self.complete = True
return ctxt
def _assess_status():
"""Assess status of relations and services for local unit"""
if is_flag_set('snap.channel.invalid'):
status_set(
'blocked', 'Invalid snap channel '
'configured: {}'.format(config('channel')))
return
if is_flag_set('config.dns_vip.invalid'):
status_set('blocked', 'vip and dns-ha-access-record configured')
return
if is_flag_set('config.lb_vip.invalid'):
status_set('blocked', 'lb-provider and vip are mutually exclusive')
return
if is_flag_set('config.lb_dns.invalid'):
status_set(
'blocked', 'lb-provider and dns-ha-access-record are '
'mutually exclusive')
return
if unitdata.kv().get('charm.vault.series-upgrading'):
status_set(
"blocked", "Ready for do-release-upgrade and reboot. "
"Set complete when finished.")
return
if is_flag_set('failed.to.start'):
status_set("blocked",
"Vault failed to start; check journalctl -u vault")
return
_missing_interfaces = []
_incomplete_interfaces = []
_assess_interface_groups(REQUIRED_INTERFACES,
optional=False,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
if _missing_interfaces or _incomplete_interfaces:
state = 'blocked' if _missing_interfaces else 'waiting'
status_set(state,
', '.join(_missing_interfaces + _incomplete_interfaces))
return
health = None
if service_running('vault'):
try:
health = vault.get_vault_health()
except Exception:
log(traceback.format_exc(), level=ERROR)
status_set('blocked', 'Vault health check failed')
return
else:
status_set('blocked', 'Vault service not running')
return
if health.get('version'):
application_version_set(health.get('version'))
else:
application_version_set('Unknown')
status_set('blocked', 'Unknown vault version')
return
if not health['initialized']:
status_set('blocked', 'Vault needs to be initialized')
return
if health['sealed']:
status_set('blocked', 'Unit is sealed')
return
if not leader_get(vault.CHARM_ACCESS_ROLE_ID):
status_set(
'blocked',
'Vault charm not yet authorized: run authorize-charm action.')
return
if not client_approle_authorized():
status_set('blocked', 'Vault cannot authorize approle')
return
lb_provider = endpoint_from_name('lb-provider')
is_leader = is_flag_set('leadership.is_leader')
if is_leader and lb_provider and lb_provider.is_available:
if not lb_provider.has_response:
status_set('waiting', 'Waiting for load balancer')
return
response = lb_provider.get_response('vault')
if response.error:
status_set(
'blocked', 'Load balancer failed: '
'{}'.format(response.error_message or response.error_fields))
return
is_leader = is_flag_set('leadership.is_leader')
has_ca = is_flag_set('charm.vault.ca.ready')
has_cert_reqs = is_flag_set('certificates.certs.requested')
if is_leader and has_cert_reqs and not has_ca:
status_set('blocked', 'Missing CA cert')
return
has_certs_relation = is_flag_set('certificates.available')
if is_leader and has_certs_relation and not has_ca:
status_set('blocked', 'Missing CA cert')
return
_assess_interface_groups(OPTIONAL_INTERFACES,
optional=True,
missing_interfaces=_missing_interfaces,
incomplete_interfaces=_incomplete_interfaces)
if _missing_interfaces or _incomplete_interfaces:
state = 'blocked' if _missing_interfaces else 'waiting'
status_set(state,
', '.join(_missing_interfaces + _incomplete_interfaces))
return
mlock_disabled = is_container() or config('disable-mlock')
vault_installed_version = snap.get_installed_version('vault')
vault_running_version = health.get('version')
if vault_installed_version != vault_running_version:
status_set(
'active',
'New version of vault installed, manual intervention required '
'to restart the service.')
return
if is_flag_set('etcd.tls.available'):
client = vault.get_local_client()
if not client.ha_status['ha_enabled']:
status_set(
'active',
'Vault running as non-HA, manual intervention required '
'to restart the service.')
return
status_set(
'active', 'Unit is ready '
'(active: {}, mlock: {})'.format(
str(not health['standby']).lower(),
'disabled' if mlock_disabled else 'enabled'))
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# pylint: disable=c0111,c0103,c0301,c0412,e0401
import os
import subprocess as sp
import json
from charms.reactive import when, when_not, set_state
from charmhelpers.core.templating import render
from charmhelpers.core import unitdata
from charmhelpers.core.hookenv import status_set, service_name
unitd = unitdata.kv()
FLOW_PATH = '/opt/dataflow'
@when('dataflow.available')
@when_not('activemq_dataflow.installed')
def install_activemq_dataflow(dataflow):
set_state('activemq_dataflow.connected')
unitd.set('nodes', ['node-red-node-mongodb', 'node-red-node-stomp'])
status_set('blocked', 'Waiting for a relation with ActiveMQ and MongoDB')
################################################################################
# First Relation with Db then ActiveMQ
################################################################################
@when('db.available', 'activemq_dataflow.connected')
@when_not('activemq_dataflow.dbconnected')
def connect_to_db(db):
def get_local(self, key, default=None):
"""
Retrieve some data previously set via :meth:`set_local` for this conversation.
"""
key = '%s.%s.%s' % (self.key, 'local-data', key)
return unitdata.kv().get(key, default)
def configure_local_ephemeral_storage():
"""Configure local block device for use as ephemeral instance storage"""
# Preflight check vault relation if encryption is enabled
vault_kv = vaultlocker.VaultKVContext(
secret_backend=vaultlocker.VAULTLOCKER_BACKEND)
context = vault_kv()
encrypt = config('encrypt')
if encrypt and not vault_kv.complete:
log("Encryption requested but vault relation not complete",
level=DEBUG)
return
elif encrypt and vault_kv.complete:
# NOTE: only write vaultlocker configuration once relation is complete
# otherwise we run the chance of an empty configuration file
# being installed on a machine with other vaultlocker based
# services
vaultlocker.write_vaultlocker_conf(context, priority=80)
db = kv()
storage_configured = db.get('storage-configured', False)
if storage_configured:
log("Ephemeral storage already configured, skipping", level=DEBUG)
return
dev = determine_block_device()
if not dev:
log('No block device configuration found, skipping', level=DEBUG)
return
if not is_block_device(dev):
log("Device '{}' is not a block device, "
"unable to configure storage".format(dev),
level=DEBUG)
return
# NOTE: this deals with a dm-crypt'ed block device already in
# use
if is_device_mounted(dev):
log("Device '{}' is already mounted, "
"unable to configure storage".format(dev),
level=DEBUG)
return
options = None
if encrypt:
dev_uuid = str(uuid.uuid4())
check_call(['vaultlocker', 'encrypt', '--uuid', dev_uuid, dev])
dev = '/dev/mapper/crypt-{}'.format(dev_uuid)
options = ','.join([
"defaults",
"nofail",
("x-systemd.requires="
"vaultlocker-decrypt@{uuid}.service".format(uuid=dev_uuid)),
"comment=vaultlocker",
])
# If not cleaned and in use, mkfs should fail.
mkfs_xfs(dev, force=True)
mountpoint = '/var/lib/nova/instances'
filesystem = "xfs"
mount(dev, mountpoint, filesystem=filesystem)
fstab_add(dev, mountpoint, filesystem, options=options)
check_call(['chown', '-R', 'nova:nova', mountpoint])
check_call(['chmod', '-R', '0755', mountpoint])
# NOTE: record preparation of device - this ensures that ephemeral
# storage is never reconfigured by mistake, losing instance disks
db.set('storage-configured', True)
db.flush()
def stop(self):
flink_appID = unitdata.kv().get('flink.ID')
if flink_appID:
utils.run_as('flink', 'yarn', 'application', '-kill', flink_appID)
return
def _save(self):
if not self:
unitdata.kv().unset(self._cache_key)
else:
unitdata.kv().set(self._cache_key,
[item._serialize() for item in self])
def prepare_disks_and_activate():
if use_vaultlocker():
# NOTE: vault/vaultlocker preflight check
vault_kv = vaultlocker.VaultKVContext(vaultlocker.VAULTLOCKER_BACKEND)
context = vault_kv()
if not vault_kv.complete:
log('Deferring OSD preparation as vault not ready', level=DEBUG)
return
else:
log('Vault ready, writing vaultlocker configuration', level=DEBUG)
vaultlocker.write_vaultlocker_conf(context)
osd_journal = get_journal_devices()
if not osd_journal.isdisjoint(set(get_devices())):
raise ValueError('`osd-journal` and `osd-devices` options must not'
'overlap.')
log("got journal devs: {}".format(osd_journal), level=DEBUG)
# pre-flight check of eligible device pristinity
devices = get_devices()
# if a device has been previously touched we need to consider it as
# non-pristine. If it needs to be re-processed it has to be zapped
# via the respective action which also clears the unitdata entry.
db = kv()
touched_devices = db.get('osd-devices', [])
devices = [dev for dev in devices if dev not in touched_devices]
log('Skipping osd devices previously processed by this unit: {}'.format(
touched_devices))
# filter osd-devices that are file system paths
devices = [dev for dev in devices if dev.startswith('/dev')]
# filter osd-devices that does not exist on this unit
devices = [dev for dev in devices if os.path.exists(dev)]
# filter osd-devices that are already mounted
devices = [dev for dev in devices if not is_device_mounted(dev)]
# filter osd-devices that are active bluestore devices
devices = [
dev for dev in devices if not ceph.is_active_bluestore_device(dev)
]
log('Checking for pristine devices: "{}"'.format(devices), level=DEBUG)
if not all(ceph.is_pristine_disk(dev) for dev in devices):
status_set(
'blocked', 'Non-pristine devices detected, consult '
'`list-disks`, `zap-disk` and `blacklist-*` actions.')
return
if is_osd_bootstrap_ready():
log('ceph bootstrapped, rescanning disks')
emit_cephconf()
bluestore = use_bluestore()
ceph.udevadm_settle()
for dev in get_devices():
ceph.osdize(dev, config('osd-format'), osd_journal,
config('ignore-device-errors'), config('osd-encrypt'),
bluestore, config('osd-encrypt-keymanager'))
# Make it fast!
if config('autotune'):
log(
'The autotune config is deprecated and planned '
'for removal in the next release.',
level=WARNING)
ceph.tune_dev(dev)
ceph.start_osds(get_devices())
# Notify MON cluster as to how many OSD's this unit bootstrapped
# into the cluster
for r_id in relation_ids('mon'):
relation_set(relation_id=r_id,
relation_settings={
'bootstrapped-osds':
len(db.get('osd-devices', [])),
'ceph_release':
ceph.resolve_ceph_version(
hookenv.config('source') or 'distro')
})
def amqp_changed(relation_id=None,
remote_unit=None,
check_deferred_restarts=True):
"""Update amqp relations.
:param relation_id: Relation id to update
:type relation_id: str
:param remote_unit: Remote unit on relation_id to update
:type remote_unit: str
:param check_deferred_events: Whether to check if restarts are
permitted before running hook.
:type check_deferred_events: bool
"""
allowed, reason = is_hook_allowed(
'amqp-relation-changed',
check_deferred_restarts=check_deferred_restarts)
if not allowed:
log(reason, "WARN")
return
singleset = set(['username', 'vhost'])
host_addr = ch_ip.get_relation_ip(
rabbit_net_utils.AMQP_INTERFACE,
cidr_network=config(rabbit_net_utils.AMQP_OVERRIDE_CONFIG))
sent_update = False
if rabbit.leader_node_is_ready():
relation_settings = {
'hostname': host_addr,
'private-address': host_addr
}
# NOTE: active/active case
if config('prefer-ipv6'):
relation_settings['private-address'] = host_addr
current = relation_get(rid=relation_id, unit=remote_unit)
if singleset.issubset(current):
if not all([current.get('username'), current.get('vhost')]):
log('Relation not ready.', DEBUG)
return
# Provide credentials to relations. If password is already
# available on peer relation then use it instead of reconfiguring.
username = current['username']
vhost = current['vhost']
admin = current.get('admin', False)
ttlname = current.get('ttlname')
ttlreg = current.get('ttlreg')
ttl = current.get('ttl')
amqp_rid = relation_id or get_relation_id()
password = configure_amqp(username,
vhost,
amqp_rid,
admin=admin,
ttlname=ttlname,
ttlreg=ttlreg,
ttl=ttl)
relation_settings['password'] = password
else:
# NOTE(hopem): we should look at removing this code since i don't
# think it's ever used anymore and stems from the days
# when we needed to ensure consistency between
# peerstorage (replaced by leader get/set) and amqp
# relations.
queues = {}
for k, v in current.items():
amqp_rid = k.split('_')[0]
x = '_'.join(k.split('_')[1:])
if amqp_rid not in queues:
queues[amqp_rid] = {}
queues[amqp_rid][x] = v
for amqp_rid in queues:
if singleset.issubset(queues[amqp_rid]):
username = queues[amqp_rid]['username']
vhost = queues[amqp_rid]['vhost']
ttlname = queues[amqp_rid].get('ttlname')
ttlreg = queues[amqp_rid].get('ttlreg')
ttl = queues[amqp_rid].get('ttl')
password = configure_amqp(username,
vhost,
amqp_rid,
admin=admin,
ttlname=ttlname,
ttlreg=ttlreg,
ttl=ttl)
key = '_'.join([amqp_rid, 'password'])
relation_settings[key] = password
ssl_utils.configure_client_ssl(relation_settings)
if is_clustered():
relation_settings['clustered'] = 'true'
# NOTE(dosaboy): this stanza can be removed once we fully remove
# deprecated HA support.
if is_relation_made('ha'):
# active/passive settings
relation_settings['vip'] = config('vip')
# or ha-vip-only to support active/active, but
# accessed via a VIP for older clients.
if config('ha-vip-only') is True:
relation_settings['ha-vip-only'] = 'true'
# set if need HA queues or not
if cmp_pkgrevno('rabbitmq-server', '3.0.1') < 0:
relation_settings['ha_queues'] = True
log(
"Updating relation {} keys {}".format(
relation_id or get_relation_id(),
','.join(relation_settings.keys())), DEBUG)
peer_store_and_set(relation_id=relation_id,
relation_settings=relation_settings)
sent_update = True
elif not is_leader() and rabbit.client_node_is_ready():
if not rabbit.clustered():
log("This node is not clustered yet, defer sending data to client",
level=DEBUG)
return
log("Propagating peer settings to all amqp relations", DEBUG)
# NOTE(jamespage) clear relation to deal with data being
# removed from peer storage.
relation_clear(relation_id)
# Each unit needs to set the db information otherwise if the unit
# with the info dies the settings die with it Bug# 1355848
for rel_id in relation_ids('amqp'):
peerdb_settings = peer_retrieve_by_prefix(rel_id)
if 'password' in peerdb_settings:
peerdb_settings['hostname'] = host_addr
peerdb_settings['private-address'] = host_addr
relation_set(relation_id=rel_id, **peerdb_settings)
sent_update = True
kvstore = kv()
update_done = kvstore.get(INITIAL_CLIENT_UPDATE_KEY, False)
if sent_update and not update_done:
kvstore.set(key=INITIAL_CLIENT_UPDATE_KEY, value=True)
kvstore.flush()
def configure_amqp(username,
vhost,
relation_id,
admin=False,
ttlname=None,
ttlreg=None,
ttl=None):
"""Configure rabbitmq server.
This function creates user/password, vhost and sets user permissions. It
also enabales mirroring queues if requested.
Calls to rabbitmqctl are costly and as such we aim to limit them by only
doing them if we detect that a settings needs creating or updating. To
achieve this we track what we set by storing key/value pairs associated
with a particular relation id in a local database.
Since this function is only supposed to be called by the cluster leader,
the database is expected to be invalidated if it exists and we are no
longer leader so as to ensure that a leader switch results in a
rabbitmq configuraion consistent with the current leader's view.
:param username: client username.
:param vhost: vhost name.
:param relation_id: optional relation id used to identify the context of
this operation. This should always be provided
so that we can track what has been set.
:param admin: boolean value defining whether the new user is admin.
:param ttlname: the name of ttl
:param ttlreg: the regular expression of ttl
:param ttl: the vaule of ttl
:returns: user password
"""
log(
"Configuring rabbitmq for user '{}' vhost '{}' (rid={})".format(
username, vhost, relation_id), DEBUG)
if not relation_id:
raise Exception("Invalid relation id '{}' provided to "
"{}()".format(relation_id, configure_amqp.__name__))
# get and update service password
password = rabbit.get_rabbit_password(username)
expected = {
'username': username,
'vhost': vhost,
'ttl': ttl,
'mirroring-queues': config('mirroring-queues')
}
kvstore = kv()
tracker = kvstore.get('amqp_config_tracker') or {}
val = tracker.get(relation_id)
if val == expected and not val.get('stale'):
log(
"Rabbit already configured for relation "
"'{}'".format(relation_id), DEBUG)
return password
else:
tracker[relation_id] = expected
# update vhost
rabbit.create_vhost(vhost)
# NOTE(jamespage): Workaround until we have a good way
# of generally disabling notifications
# based on which services are deployed.
if vhost == 'openstack':
rabbit.configure_notification_ttl(vhost, config('notification-ttl'))
rabbit.configure_ttl(vhost, ttlname, ttlreg, ttl)
if admin:
rabbit.create_user(username, password, ['administrator'])
else:
rabbit.create_user(username, password)
rabbit.grant_permissions(username, vhost)
# NOTE(freyes): after rabbitmq-server 3.0 the method to define HA in the
# queues is different
# http://www.rabbitmq.com/blog/2012/11/19/breaking-things-with-rabbitmq-3-0
if config('mirroring-queues'):
rabbit.set_ha_mode(vhost, 'all')
kvstore.set(key='amqp_config_tracker', value=tracker)
kvstore.flush()
return password
def configure(self, available_hosts, zk_units, peers):
"""
This is the core logic of setting up spark.
Two flags are needed:
* Namenode exists aka HDFS is ready
* Resource manager exists aka YARN is ready
both flags are infered from the available hosts.
:param dict available_hosts: Hosts that Spark should know about.
"""
# Bootstrap spark
if not unitdata.kv().get('spark.bootstrapped', False):
self.setup()
unitdata.kv().set('spark.bootstrapped', True)
# Set KV based on connected applications
unitdata.kv().set('zookeeper.units', zk_units)
unitdata.kv().set('sparkpeer.units', peers)
unitdata.kv().flush(True)
# Get our config ready
dc = self.dist_config
events_log_dir = 'file://{}'.format(dc.path('spark_events'))
mode = hookenv.config()['spark_execution_mode']
master_ip = utils.resolve_private_address(available_hosts['spark-master'])
master_url = self.get_master_url(master_ip)
# Setup hosts dict
hosts = {
'spark': master_ip,
}
if 'namenode' in available_hosts:
hosts['namenode'] = available_hosts['namenode']
events_log_dir = self.setup_hdfs_logs()
if 'resourcemanager' in available_hosts:
hosts['resourcemanager'] = available_hosts['resourcemanager']
# Setup roles dict. We always include the history server and client.
# Determine other roles based on our execution mode.
roles = ['spark-history-server', 'spark-client']
if mode == 'standalone':
roles.append('spark-master')
roles.append('spark-worker')
elif mode.startswith('yarn'):
roles.append('spark-on-yarn')
roles.append('spark-yarn-slave')
# Setup overrides dict
override = {
'spark::common::master_url': master_url,
'spark::common::event_log_dir': events_log_dir,
'spark::common::history_log_dir': events_log_dir,
}
if zk_units:
zks = []
for unit in zk_units:
ip = utils.resolve_private_address(unit['host'])
zks.append("%s:%s" % (ip, unit['port']))
zk_connect = ",".join(zks)
override['spark::common::zookeeper_connection_string'] = zk_connect
else:
override['spark::common::zookeeper_connection_string'] = None
# Create our site.yaml and trigger puppet
bigtop = Bigtop()
bigtop.render_site_yaml(hosts, roles, override)
bigtop.trigger_puppet()
# Do this after our puppet bits in case puppet overrides needed perms
if 'namenode' not in available_hosts:
# Local event dir (not in HDFS) needs to be 777 so non-spark
# users can write job history there. It needs to be g+s so
# all entries will be readable by spark (in the spark group).
# It needs to be +t so users cannot remove files they don't own.
dc.path('spark_events').chmod(0o3777)
self.patch_worker_master_url(master_ip, master_url)
# handle tuning options that may be set as percentages
driver_mem = '1g'
req_driver_mem = hookenv.config()['driver_memory']
executor_mem = '1g'
req_executor_mem = hookenv.config()['executor_memory']
if req_driver_mem.endswith('%'):
if mode == 'standalone' or mode.startswith('local'):
mem_mb = host.get_total_ram() / 1024 / 1024
req_percentage = float(req_driver_mem.strip('%')) / 100
driver_mem = str(int(mem_mb * req_percentage)) + 'm'
else:
hookenv.log("driver_memory percentage in non-local mode. Using 1g default.",
level=None)
else:
driver_mem = req_driver_mem
if req_executor_mem.endswith('%'):
if mode == 'standalone' or mode.startswith('local'):
mem_mb = host.get_total_ram() / 1024 / 1024
req_percentage = float(req_executor_mem.strip('%')) / 100
executor_mem = str(int(mem_mb * req_percentage)) + 'm'
else:
hookenv.log("executor_memory percentage in non-local mode. Using 1g default.",
level=None)
else:
executor_mem = req_executor_mem
spark_env = '/etc/spark/conf/spark-env.sh'
utils.re_edit_in_place(spark_env, {
r'.*SPARK_DRIVER_MEMORY.*': 'export SPARK_DRIVER_MEMORY={}'.format(driver_mem),
r'.*SPARK_EXECUTOR_MEMORY.*': 'export SPARK_EXECUTOR_MEMORY={}'.format(executor_mem),
}, append_non_matches=True)
# Install SB (subsequent calls will reconfigure existing install)
# SparkBench looks for the spark master in /etc/environment
with utils.environment_edit_in_place('/etc/environment') as env:
env['MASTER'] = master_url
self.install_benchmark()
def load(cls, cache_key, deserializer, key_attr):
"""
Load the persisted cache and return a new instance of this class.
"""
items = unitdata.kv().get(cache_key) or []
return cls(cache_key, [deserializer(item) for item in items], key_attr)
def unset_meta_generation():
unitdata.kv().unset(kvdata.KEY_META_GENERATION)
def clear(self):
"""
Clear this request's cached data.
"""
unitdata.kv().unset(self._hash_key)
def get_meta_generation():
return unitdata.kv().get(kvdata.KEY_META_GENERATION)
def is_installed(self):
return unitdata.kv().get('flink.installed')
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from charms.reactive import Endpoint, toggle_flag, set_flag, data_changed
from charmhelpers.core import hookenv, unitdata
DB = unitdata.kv()
class KubeControlProvider(Endpoint):
"""
Implements the kubernetes-master side of the kube-control interface.
"""
def manage_flags(self):
toggle_flag(self.expand_name("{endpoint_name}.connected"),
self.is_joined)
toggle_flag(
self.expand_name("{endpoint_name}.gpu.available"),
self.is_joined and self._get_gpu(),
)
requests_data_id = self.expand_name("{endpoint_name}.requests")
requests = self.auth_user()
def _load_creds():
return kv().get('charm.vsphere.full-creds')
def configure_registry():
'''Recreate the docker registry config.yml.'''
charm_config = hookenv.config()
registry_config = {'version': '0.1'}
registry_config_file = '/etc/docker/registry/config.yml'
# Some things need to be volume mounted in the container. Keep track of
# those (recreate each time we configure). Regardless of the src location,
# we explicitly mount config in the container under /etc/docker/registry.
kv = unitdata.kv()
kv.unset('docker_volumes')
docker_volumes = {registry_config_file: '/etc/docker/registry/config.yml'}
# auth (https://docs.docker.com/registry/configuration/#auth)
auth = {}
auth_basic = _get_auth_basic()
if auth_basic:
auth['htpasswd'] = auth_basic
docker_volumes[auth_basic['path']] = '/etc/docker/registry/htpasswd'
auth_token = _get_auth_token()
if auth_token:
auth['token'] = auth_token
docker_volumes[auth_token['rootcertbundle']] = '/etc/docker/registry/auth_token.pem'
registry_config['auth'] = auth
# http (https://docs.docker.com/registry/configuration/#http)
port = charm_config.get('registry-port')
http = {'addr': '0.0.0.0:{}'.format(port),
'headers': {'X-Content-Type-Options': ['nosniff']},
'relativeurls': True}
if charm_config.get('http-host'):
http['host'] = charm_config['http-host']
http_secret = leader_get('http-secret')
if http_secret:
http['secret'] = http_secret
# Only does anything if tls-*-blob set.
_write_tls_blobs_to_files()
tls_ca = charm_config.get('tls-ca-path', '')
tls_cert = charm_config.get('tls-cert-path', '')
tls_key = charm_config.get('tls-key-path', '')
if os.path.isfile(tls_cert) and os.path.isfile(tls_key):
http['tls'] = {
'certificate': tls_cert,
'key': tls_key,
}
docker_volumes[tls_cert] = '/etc/docker/registry/registry.crt'
docker_volumes[tls_key] = '/etc/docker/registry/registry.key'
if os.path.isfile(tls_ca):
http['tls']['clientcas'] = [tls_ca]
docker_volumes[tls_ca] = '/etc/docker/registry/ca.crt'
registry_config['http'] = http
# log (https://docs.docker.com/registry/configuration/#log)
registry_config['log'] = {
'level': charm_config['log-level'],
'formatter': 'json',
'fields': {
'service': 'registry',
}
}
# health (https://docs.docker.com/registry/configuration/#health)
registry_config['health'] = {
'storagedriver': {
'enabled': True,
'interval': '10s',
'threshold': 3,
}
}
# storage (https://docs.docker.com/registry/configuration/#storage)
# we must have 1 (and only 1) storage driver
storage = {}
if charm_config.get('storage-swift-authurl'):
storage['swift'] = {
'authurl': charm_config.get('storage-swift-authurl', ''),
'username': charm_config.get('storage-swift-username', ''),
'password': charm_config.get('storage-swift-password', ''),
'region': charm_config.get('storage-swift-region', ''),
'container': charm_config.get('storage-swift-container', ''),
'tenant': charm_config.get('storage-swift-tenant', ''),
}
# Openstack Domain settings (https://github.com/docker/docker.github.io/blob/master/registry/storage-drivers/swift.md)
val = charm_config.get('storage-swift-domain', '')
if val is not '':
storage['swift'].update({'domain': val})
storage['redirect'] = {'disable': True}
else:
# If we're not swift, we're local.
container_registry_path = '/var/lib/registry'
storage['filesystem'] = {'rootdirectory': container_registry_path}
storage['cache'] = {'blobdescriptor': 'inmemory'}
# Local storage is mounted from the host so images persist across
# registry container restarts.
host_registry_path = '/srv/registry'
os.makedirs(host_registry_path, exist_ok=True)
docker_volumes[host_registry_path] = container_registry_path
if charm_config.get('storage-delete'):
storage['delete'] = {'enabled': True}
if charm_config.get('storage-read-only'):
storage['maintenance'] = {'readonly': {'enabled': True}}
registry_config['storage'] = storage
os.makedirs(os.path.dirname(registry_config_file), exist_ok=True)
host.write_file(
registry_config_file,
yaml.safe_dump(registry_config),
perms=0o600,
)
# NB: all hooks will flush, but do an explicit one now in case we call
# something that needs this data before our hook ends.
kv.set('docker_volumes', docker_volumes)
kv.flush(True)
# Configure the system so our local 'docker' commands can interact
# with the registry.
_configure_local_client()
import shutil
from subprocess import check_call
from charmhelpers.core import hookenv
from charmhelpers.core import unitdata
from charms import reactive
from charms.reactive import hook
from charms.reactive import when, when_not, when_any
db = unitdata.kv()
config = hookenv.config()
@hook('config-changed')
def config_changed():
restart_if_need()
@hook('mysql-relation-changed')
def mysql_changed():
restart_if_need()
def restart_if_need():
if reactive.is_state('restcomm.started'):
reactive.set_state('restcomm.changed')
@hook('api-relation-joined')
def get_blacklist():
"""Get blacklist stored in the local kv() store"""
db = unitdata.kv()
return db.get('osd-blacklist', [])
#python3 pylint:disable=c0111
import json
import socket
from charmhelpers.core import unitdata
from charms.reactive import hook
from charms.reactive import RelationBase
from charms.reactive import scopes
RANGE = 29000
KV = unitdata.kv()
class OpenedPortsRequires(RelationBase):
scope = scopes.UNIT
@hook('{requires:opened-ports}-relation-{joined,changed}')
def changed(self):
conv = self.conversation()
if conv.get_remote('opened-ports'):
# this unit's conversation has a port, so
# it is part of the set of available units
opened_ports = json.loads(conv.get_remote('opened-ports'))
port_forwards = conv.get_local('port-forwards', [])
# Get public ip address
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.connect(("google.com", 80))
public_address = sock.getsockname()[0]
sock.close()
def configure(self, available_hosts, zk_units, peers, extra_libs):
"""
This is the core logic of setting up spark.
:param dict available_hosts: Hosts that Spark should know about.
:param list zk_units: List of Zookeeper dicts with host/port info.
:param list peers: List of Spark peer tuples (unit name, IP).
:param list extra_libs: List of extra lib paths for driver/executors.
"""
# Set KV based on connected applications
unitdata.kv().set('zookeeper.units', zk_units)
unitdata.kv().set('sparkpeer.units', peers)
unitdata.kv().flush(True)
# Get our config ready
dc = self.dist_config
mode = hookenv.config()['spark_execution_mode']
master_ip = utils.resolve_private_address(
available_hosts['spark-master'])
master_url = self.get_master_url(master_ip)
req_driver_mem = hookenv.config()['driver_memory']
req_executor_mem = hookenv.config()['executor_memory']
if mode.startswith('yarn'):
spark_events = 'hdfs://{}'.format(dc.path('spark_events'))
else:
spark_events = 'file://{}'.format(dc.path('spark_events'))
# handle tuning options that may be set as percentages
driver_mem = '1g'
executor_mem = '1g'
if req_driver_mem.endswith('%'):
if mode == 'standalone' or mode.startswith('local'):
mem_mb = host.get_total_ram() / 1024 / 1024
req_percentage = float(req_driver_mem.strip('%')) / 100
driver_mem = str(int(mem_mb * req_percentage)) + 'm'
else:
hookenv.log(
"driver_memory percentage in non-local mode. "
"Using 1g default.",
level=hookenv.WARNING)
else:
driver_mem = req_driver_mem
if req_executor_mem.endswith('%'):
if mode == 'standalone' or mode.startswith('local'):
mem_mb = host.get_total_ram() / 1024 / 1024
req_percentage = float(req_executor_mem.strip('%')) / 100
executor_mem = str(int(mem_mb * req_percentage)) + 'm'
else:
hookenv.log(
"executor_memory percentage in non-local mode. "
"Using 1g default.",
level=hookenv.WARNING)
else:
executor_mem = req_executor_mem
# Some spark applications look for envars in /etc/environment
with utils.environment_edit_in_place('/etc/environment') as env:
env['MASTER'] = master_url
env['SPARK_HOME'] = dc.path('spark_home')
# Setup hosts dict
hosts = {
'spark': master_ip,
}
if 'namenode' in available_hosts:
hosts['namenode'] = available_hosts['namenode']
if 'resourcemanager' in available_hosts:
hosts['resourcemanager'] = available_hosts['resourcemanager']
# Setup roles dict. We always include the history server and client.
# Determine other roles based on our execution mode.
roles = ['spark-history-server', 'spark-client']
if mode == 'standalone':
roles.append('spark-master')
roles.append('spark-worker')
elif mode.startswith('yarn'):
roles.append('spark-on-yarn')
roles.append('spark-yarn-slave')
# Setup overrides dict
override = {
'spark::common::master_url':
master_url,
'spark::common::event_log_dir':
spark_events,
'spark::common::history_log_dir':
spark_events,
'spark::common::extra_lib_dirs':
':'.join(extra_libs) if extra_libs else None,
'spark::common::driver_mem':
driver_mem,
'spark::common::executor_mem':
executor_mem,
}
if zk_units:
zks = []
for unit in zk_units:
ip = utils.resolve_private_address(unit['host'])
zks.append("%s:%s" % (ip, unit['port']))
zk_connect = ",".join(zks)
override['spark::common::zookeeper_connection_string'] = zk_connect
else:
override['spark::common::zookeeper_connection_string'] = None
# Create our site.yaml and trigger puppet.
# NB: during an upgrade, we configure the site.yaml, but do not
# trigger puppet. The user must do that with the 'reinstall' action.
bigtop = Bigtop()
bigtop.render_site_yaml(hosts, roles, override)
if unitdata.kv().get('spark.version.repo', False):
hookenv.log(
"An upgrade is available and the site.yaml has been "
"configured. Run the 'reinstall' action to continue.",
level=hookenv.INFO)
else:
bigtop.trigger_puppet()
self.patch_worker_master_url(master_ip, master_url)
# Packages don't create the event dir by default. Do it each time
# spark is (re)installed to ensure location/perms are correct.
self.configure_events_dir(mode)
# Handle examples and Spark-Bench. Do this each time this method is
# called in case we need to act on a new resource or user config.
self.configure_examples()
self.configure_sparkbench()
def configure_sparkbench(self):
"""
Install/configure/remove Spark-Bench based on user config.
If config[spark_bench_enabled], fetch, install, and configure
Spark-Bench on initial invocation. Subsequent invocations will skip the
fetch/install, but will reconfigure Spark-Bench since we may need to
adjust the data dir (eg: benchmark data is stored in hdfs when spark
is in yarn mode; locally in all other execution modes).
"""
install_sb = hookenv.config()['spark_bench_enabled']
sb_dir = '/home/ubuntu/SparkBench'
if install_sb:
# Fetch/install on our first go-round, then set unit data so we
# don't reinstall every time this function is called.
if not unitdata.kv().get('spark_bench.installed', False):
sb_url = hookenv.config()['spark_bench_url']
Path(sb_dir).rmtree_p()
au = ArchiveUrlFetchHandler()
au.install(sb_url, '/home/ubuntu')
# NB: This block is unused when using one of our sb tgzs. It
# may come in handy if people want a tgz that does not expand
# to our expected sb_dir.
# #####
# Handle glob if we use a .tgz that doesn't expand to sb_dir
# sb_archive_dir = glob('/home/ubuntu/SparkBench*')[0]
# SparkBench expects to live in ~/SparkBench, so put it there
# Path(sb_archive_dir).rename(sb_dir)
# #####
# Ensure users in the spark group can write to any subdirectory
# of sb_dir (spark needs to write benchmark output there when
# running in local modes).
host.chownr(Path(sb_dir), 'ubuntu', 'spark', chowntopdir=True)
for r, d, f in os.walk(sb_dir):
os.chmod(r, 0o2775)
unitdata.kv().set('spark_bench.installed', True)
unitdata.kv().flush(True)
# Configure the SB env every time this function is called.
sb_conf = '{}/conf'.format(sb_dir)
sb_env = Path(sb_conf) / 'env.sh'
if not sb_env.exists():
(Path(sb_conf) / 'env.sh.template').copy(sb_env)
# NB: A few notes on configuring SparkBench:
# 1. Input data has been pregenerated and packed into the tgz. All
# spark cluster members will have this data locally, which enables
# us to execute benchmarks in the absense of HDFS. When spark is in
# yarn mode, we'll need to generate and store this data in HDFS
# so nodemanagers can access it (NMs obviously won't have SB
# installed locally). Set DATA_HDFS to a local dir or common HDFS
# location depending on our spark execution mode.
#
# 2. SB tries to SSH to spark workers to purge vmem caches. This
# isn't possible in containers, nor is it possible in our env
# because we don't distribute ssh keys among cluster members.
# Set MC_LIST to an empty string to prevent this behavior.
#
# 3. Throughout SB, HADOOP_HOME/bin is used as the prefix for the
# hdfs command. Bigtop's hdfs lives at /usr/bin/hdfs, so set the
# SB HADOOP_HOME accordingly (it's not used for anything else).
#
# 4. Use our MASTER envar to set the SparkBench SPARK_MASTER url.
# It is updated every time we (re)configure spark.
mode = hookenv.config()['spark_execution_mode']
if mode.startswith('yarn'):
sb_data_dir = "hdfs:///user/ubuntu/SparkBench"
else:
sb_data_dir = "file://{}".format(sb_dir)
utils.re_edit_in_place(
sb_env, {
r'^DATA_HDFS *=.*': 'DATA_HDFS="{}"'.format(sb_data_dir),
r'^DATASET_DIR *=.*':
'DATASET_DIR="{}/dataset"'.format(sb_dir),
r'^MC_LIST *=.*': 'MC_LIST=""',
r'.*HADOOP_HOME *=.*': 'HADOOP_HOME="/usr"',
r'.*SPARK_HOME *=.*': 'SPARK_HOME="/usr/lib/spark"',
r'^SPARK_MASTER *=.*': 'SPARK_MASTER="$MASTER"',
})
else:
# config[spark_bench_enabled] is false; remove it
Path(sb_dir).rmtree_p()
unitdata.kv().set('spark_bench.installed', False)
unitdata.kv().flush(True)
def service_cidr():
''' Return the charm's service-cidr config '''
db = unitdata.kv()
frozen_cidr = db.get('kubernetes-master.service-cidr')
return frozen_cidr or hookenv.config('service-cidr')
def post_series_upgrade():
"""Handler for post-series-upgrade.
"""
unitdata.kv().set('charm.vault.series-upgrading', False)
def freeze_service_cidr():
''' Freeze the service CIDR. Once the apiserver has started, we can no
longer safely change this value. '''
db = unitdata.kv()
db.set('kubernetes-master.service-cidr', service_cidr())
def render_and_restart():
rabbit.ConfigRenderer(rabbit.CONFIG_FILES()).write_all()
render_and_restart()
update_clients()
@hooks.hook('update-status')
@harden()
def update_status():
log('Updating status.')
if __name__ == '__main__':
try:
hooks.execute(sys.argv)
except UnregisteredHookError as e:
log('Unknown hook {} - skipping.'.format(e))
# This solves one off problems waiting for the cluster to complete
# It will get executed only once as soon as leader_node_is_ready()
# or client_node_is_ready() returns True
# Subsequent client requests will be handled by normal
# amqp-relation-changed hooks
kvstore = kv()
if not kvstore.get(INITIAL_CLIENT_UPDATE_KEY, False):
log("Rerunning update_clients as initial update not yet performed",
level=DEBUG)
update_clients()
rabbit.assess_status(rabbit.ConfigRenderer(rabbit.CONFIG_FILES()))
def _add_override(self, name, value):
unitdata.kv().update({
name: value,
},
prefix='zeppelin.bigtop.overrides.')
