def check_relations(self):
'''
This function checks the 'additional_relations' list against the joined
relation states so we don't have to explicitly set_status in each reactive
function
'''
additional_relations = []
metadata_stream = open('metadata.yaml', 'r')
data = yaml.load(metadata_stream)
for key in data['requires']:
additional_relations.append(key)
current_relations = additional_relations
all_states = get_states()
for k, v in all_states.items():
if "joined" in k:
relname = k.split('.')[0]
if relname in additional_relations:
current_relations.remove(relname)
wait_rels = ', '.join(current_relations)
if len(current_relations) > 0:
hookenv.status_set(
'active',
'Ready. Accepting connections to {}'.format(wait_rels))
else:
hookenv.status_set('active', 'Ready')
def changed(self):
hookenv.log('Data: {}'.format({
'local_spec': self.local_spec(),
'remote_spec': self.remote_spec(),
'namenodes': self.namenodes(),
'port': self.port(),
'webhdfs_port': self.webhdfs_port(),
'hosts_map': self.hosts_map(),
'local_hostname': self.local_hostname(),
}))
conv = self.conversation()
available = all([
self.remote_spec() is not None,
self.hosts_map(),
self.namenodes(),
self.port(),
self.webhdfs_port(),
self.ssh_key()
])
spec_mismatch = available and not self._spec_match()
visible = self.local_hostname() in self.hosts_map().values()
ready = available and visible
conv.toggle_state('{relation_name}.spec.mismatch', spec_mismatch)
conv.toggle_state('{relation_name}.ready', ready and not spec_mismatch)
hookenv.log('States: {}'.format(set(get_states().keys())))
def persist_state():
"""Fake persistent state by calling helpers that modify unitdata.kv"""
states = [k for k in bus.get_states().keys()
if k.startswith('plugins') or k.startswith('extra_plugins')]
helpers.any_file_changed(telegraf.list_config_files())
if states:
helpers.data_changed('active_plugins', states)
def changed(self):
hookenv.log('Data: {}'.format({
'local_spec': self.local_spec(),
'remote_spec': self.remote_spec(),
'clustername': self.clustername(),
'namenodes': self.namenodes(),
'port': self.port(),
'webhdfs_port': self.webhdfs_port(),
'hosts_map': self.hosts_map(),
'local_hostname': self.local_hostname(),
}))
conv = self.conversation()
available = all([
self.remote_spec() is not None,
self.hosts_map(),
self.clustername(),
self.namenodes(),
self.port(),
self.webhdfs_port(),
self.ssh_key()])
spec_mismatch = available and not self._spec_match()
visible = self.local_hostname() in self.hosts_map().values()
ready = available and visible
conv.toggle_state('{relation_name}.spec.mismatch', spec_mismatch)
conv.toggle_state('{relation_name}.ready', ready and not spec_mismatch)
hookenv.log('States: {}'.format(set(get_states().keys())))
def changed(self):
conv = self.conversation()
hookenv.log('Data: {}'.format({
'remote_spec':
self.remote_spec(),
'local_spec':
self.local_spec(),
'hosts-map':
self.hosts_map(),
'resourcemanagers':
self.resourcemanagers(),
'port':
self.port(),
'hs_http':
self.hs_http(),
'hs_ipc':
self.hs_ipc(),
}))
available = all([
self.remote_spec() is not None,
self.hosts_map(),
self.resourcemanagers(),
self.port(),
self.hs_http(),
self.hs_ipc()
])
spec_mismatch = available and not self._spec_match()
ready = available and self.yarn_ready()
conv.toggle_state('{relation_name}.spec.mismatch', spec_mismatch)
conv.toggle_state('{relation_name}.ready', ready and not spec_mismatch)
hookenv.log('States: {}'.format(set(get_states().keys())))
def apply_playbook(playbook, tags=None, extra_vars=None):
# Make the default behavior to read the states from the environment
# This may not be desireable long term, TODO: investigate this
tags = tags or get_states().keys()
tags = ",".join(tags)
# translate system state to YAML inventory.
host.juju_state_to_yaml(
ansible_vars_path, namespace_separator='__',
allow_hyphens_in_keys=False, mode=(stat.S_IRUSR | stat.S_IWUSR))
# we want ansible's log output to be unbuffered
env = os.environ.copy()
env['PYTHONUNBUFFERED'] = "1"
call = [
'ansible-playbook',
'-c',
'local',
playbook,
]
if tags:
call.extend(['--tags', '{}'.format(tags)])
if extra_vars:
extra = ["%s=%s" % (k, v) for k, v in extra_vars.items()]
call.extend(['--extra-vars', " ".join(extra)])
subprocess.check_call(call, env=env)
def update_apps(self):
# Add all services disabled unless we have a joined relation
# as marked by the respective state
# Enabled by default: 'filebrowser', 'jobbrowser'
disabled_services = [
'beeswax', 'impala', 'security',
'rdbms', 'jobsub', 'pig', 'hbase', 'sqoop',
'zookeeper', 'metastore', 'spark', 'oozie', 'indexer', 'search']
for key in get_states():
if "joined" in key:
relname = key.split('.')[0]
if 'hive' in relname:
disabled_services.remove('beeswax')
disabled_services.remove('metastore')
if 'spark' in relname:
disabled_services.remove('spark')
if 'oozie' in relname:
disabled_services.remove('oozie')
if 'zookeeper' in relname:
disabled_services.remove('zookeeper')
hue_config = ''.join((self.dist_config.path('hue'), '/desktop/conf/hue.ini'))
services_string = ','.join(disabled_services)
hookenv.log("Disabled apps {}".format(services_string))
utils.re_edit_in_place(hue_config, {
r'.*app_blacklist=.*': ''.join(('app_blacklist=', services_string))
})
self.check_relations()
def changed(self):
conv = self.conversation()
hookenv.log('Data: {}'.format({
'remote_spec': self.remote_spec(),
'local_spec': self.local_spec(),
'hosts-map': self.hosts_map(),
'resourcemanagers': self.resourcemanagers(),
'port': self.port(),
'hs_http': self.hs_http(),
'hs_ipc': self.hs_ipc(),
}))
available = all([
self.remote_spec() is not None,
self.hosts_map(),
self.resourcemanagers(),
self.port(),
self.hs_http(),
self.hs_ipc()])
spec_mismatch = available and not self._spec_match()
ready = available and self.yarn_ready()
conv.toggle_state('{relation_name}.spec.mismatch', spec_mismatch)
conv.toggle_state('{relation_name}.ready', ready and not spec_mismatch)
hookenv.log('States: {}'.format(set(get_states().keys())))
def apply_playbook(playbook, tags=None, extra_vars=None):
# Make the default behavior to read the states from the environment
# This may not be desireable long term, TODO: investigate this
tags = tags or get_states().keys()
tags = ",".join(tags)
tags = "{},{}".format(tags, os.getenv('JUJU_HOOK_NAME'))
# translate system state to YAML inventory.
host.juju_state_to_yaml(ansible_vars_path,
namespace_separator='__',
allow_hyphens_in_keys=False,
mode=(stat.S_IRUSR | stat.S_IWUSR))
# we want ansible's log output to be unbuffered
env = os.environ.copy()
env['PYTHONUNBUFFERED'] = "1"
call = [
'ansible-playbook',
'-c',
'local',
playbook,
]
if tags:
call.extend(['--tags', '{}'.format(tags)])
if extra_vars:
extra = ["%s=%s" % (k, v) for k, v in extra_vars.items()]
call.extend(['--extra-vars', " ".join(extra)])
subprocess.check_call(call, env=env)
def test_basic_config(mocker, config):
service_restart = mocker.patch('reactive.telegraf.host.service_restart')
bus.set_state('telegraf.installed')
assert not base_dir().join('telegraf.conf').exists()
bus.dispatch()
assert 'telegraf.configured' in bus.get_states().keys()
assert base_dir().join('telegraf.conf').exists()
service_restart.assert_called_once_with('telegraf')
def _wrapped(*args, **kwargs):
active_states = get_states()
missing_states = [state for state in desired_states if state not in active_states]
if missing_states:
hookenv.log('%s called before state%s: %s' % (
short_action_id,
's' if len(missing_states) > 1 else '',
', '.join(missing_states)), hookenv.WARNING)
return func(*args, **kwargs)
def _wrapped(*args, **kwargs):
active_states = get_states()
missing_states = [state for state in desired_states if state not in active_states]
if missing_states:
hookenv.log(
"%s called before state%s: %s"
% (short_action_id, "s" if len(missing_states) > 1 else "", ", ".join(missing_states)),
hookenv.WARNING,
)
return func(*args, **kwargs)
def list_config_files():
config_files = [get_main_config_path()]
# only include config files for configured plugins
current_states = get_states()
for plugin in list_supported_plugins():
if 'plugins.{}.configured'.format(plugin) in current_states.keys():
config_path = '{}/{}.conf'.format(get_configs_dir(), plugin)
config_files.append(config_path)
if 'extra_plugins.configured' in current_states.keys():
config_files.append('{}/extra_plugins.conf'.format(get_configs_dir()))
return config_files
def test_config_changed_extra_options(mocker, config):
service_restart = mocker.patch('reactive.telegraf.host.service_restart')
bus.set_state('telegraf.installed')
bus.set_state('plugins.haproxy.configured')
config.save()
config.load_previous()
config['extra_options'] = yaml.dump({'inputs': {'haproxy': {'timeout': 10}}})
bus.set_state('config.changed')
bus.dispatch()
assert 'plugins.haproxy.configured' not in bus.get_states().keys()
service_restart.assert_called_once_with('telegraf')
def _wrapped(*args, **kwargs):
active_states = get_states()
missing_states = [state for state in desired_states if state not in active_states]
if missing_states:
func_id = "%s:%s:%s" % (func.__code__.co_filename,
func.__code__.co_firstlineno,
func.__code__.co_name)
hookenv.log('%s called before state%s: %s' % (
func_id,
's' if len(missing_states) > 1 else '',
', '.join(missing_states)), hookenv.WARNING)
return func(*args, **kwargs)
def start_or_restart():
states = sorted([k for k in get_states().keys()
if k.startswith('plugins') or k.startswith('extra_plugins')])
config_files_changed = helpers.any_file_changed(list_config_files())
active_plugins_changed = helpers.data_changed('active_plugins', states or '')
if config_files_changed or active_plugins_changed:
hookenv.log("Restarting telegraf")
host.service_restart('telegraf')
else:
hookenv.log("Not restarting: active_plugins_changed={} | "
"config_files_changed={}".format(active_plugins_changed,
config_files_changed))
def _migrate_conversations():
"""
Due to issue #28 (https://github.com/juju-solutions/charms.reactive/issues/28),
conversations needed to be updated to be namespaced per relation ID for SERVICE
and UNIT scope. To ensure backwards compatibility, this updates all convs in
the old format to the new.
TODO: Remove in 2.0.0
"""
for key, data in unitdata.kv().getrange('reactive.conversations.').items():
if 'local-data' in key:
continue
if 'namespace' in data:
continue
relation_name = data.pop('relation_name')
if data['scope'] == scopes.GLOBAL:
data['namespace'] = relation_name
unitdata.kv().set(key, data)
else:
# split the conv based on the relation ID
new_keys = []
for rel_id in hookenv.relation_ids(relation_name):
new_key = Conversation._key(rel_id, data['scope'])
new_units = set(hookenv.related_units(rel_id)) & set(
data['units'])
if new_units:
unitdata.kv().set(
new_key, {
'namespace': rel_id,
'scope': data['scope'],
'units': sorted(new_units),
})
new_keys.append(new_key)
unitdata.kv().unset(key)
# update the states pointing to the old conv key to point to the
# (potentially multiple) new key(s)
for state, value in get_states().items():
if not value:
continue
if key not in value['conversations']:
continue
value['conversations'].remove(key)
value['conversations'].extend(new_keys)
set_state(state, value)
def _migrate_conversations():
"""
Due to issue #28 (https://github.com/juju-solutions/charms.reactive/issues/28),
conversations needed to be updated to be namespaced per relation ID for SERVICE
and UNIT scope. To ensure backwards compatibility, this updates all convs in
the old format to the new.
TODO: Remove in 2.0.0
"""
for key, data in unitdata.kv().getrange('reactive.conversations.').items():
if 'local-data' in key:
continue
if 'namespace' in data:
continue
relation_name = data.pop('relation_name')
if data['scope'] == scopes.GLOBAL:
data['namespace'] = relation_name
unitdata.kv().set(key, data)
else:
# split the conv based on the relation ID
new_keys = []
for rel_id in hookenv.relation_ids(relation_name):
new_key = Conversation._key(rel_id, data['scope'])
new_units = set(hookenv.related_units(rel_id)) & set(data['units'])
if new_units:
unitdata.kv().set(new_key, {
'namespace': rel_id,
'scope': data['scope'],
'units': sorted(new_units),
})
new_keys.append(new_key)
unitdata.kv().unset(key)
# update the states pointing to the old conv key to point to the
# (potentially multiple) new key(s)
for state, value in get_states().items():
if not value:
continue
if key not in value['conversations']:
continue
value['conversations'].remove(key)
value['conversations'].extend(new_keys)
set_state(state, value)
def check_relations(self):
'''
This function checks the 'additional_relations' list against the joined
relation states so we don't have to explicitly set_status in each reactive
function
'''
additional_relations = []
metadata_stream = open('metadata.yaml', 'r')
data = yaml.load(metadata_stream)
for key in data['requires']:
additional_relations.append(key)
current_relations = additional_relations
all_states = get_states()
for key in all_states:
if "joined" in key:
relname = key.split('.')[0]
if relname in additional_relations:
current_relations.remove(relname)
wait_rels = ', '.join(current_relations)
if len(current_relations) > 0:
hookenv.status_set('active', 'Ready. Accepting connections to {}'.format(wait_rels))
else:
hookenv.status_set('active', 'Ready')
def waiting_db(pg):
hookenv.log("%s" % (str(get_states())))
remove_state('oostore.start')
hookenv.status_set('waiting', 'Waiting for postgresql')
def all_states(*desired_states):
"""Assert that all desired_states are active"""
active_states = get_states()
return all(state in active_states for state in desired_states)
def any_states(*desired_states):
"""Assert that any of the desired_states are active"""
active_states = get_states()
return any(state in active_states for state in desired_states)
def joined(self):
conv = self.conversation()
conv.set_state('{relation_name}.related')
hookenv.log('States: {}'.format(get_states().keys()))
def update_status():
log("States %s"%get_states() )
log("Env %s"% execution_environment() )
controllers = get_controllers()
if data_changed('controllers', controllers):
ipset_update(controllers_set_name(), controllers)
def joined(self):
conv = self.conversation()
conv.set_state('{relation_name}.related')
conv.set_remote('private-address',
ch_ip.get_relation_ip(conv.relation_name))
hookenv.log('States: {}'.format(get_states().keys()))
def missing_db():
hookenv.log("%s" % (str(get_states())))
remove_state('gobinary.start')
hookenv.status_set('blocked', 'Please add relation to mongodb')
def missing_db():
hookenv.log("%s" % (str(get_states())))
remove_state('oostore.start')
hookenv.status_set('blocked', 'Please add relation to postgresql')
hue = Hue(get_dist_config())
hue.setup_hue(namenodes, resmngmrs, hdfs_port,
yarn_port, yarn_http, yarn_ipcp)
set_state('hue.configured')
@when('hue.installed', 'hadoop.ready', 'hue.configured')
@when_not('hue.started')
def start_hue(hadoop):
hookenv.status_set('maintenance', 'Setting up Hue')
hue = Hue(get_dist_config())
hue.open_ports()
hue.start()
set_state('hue.started')
if 'hue.started' in get_states():
@when_file_changed('/etc/hue/conf/hue.ini')
def restart_hue():
# Can't seem to mix @when_file_changed and @when('hue.started')
hue = Hue(get_dist_config())
hue.restart()
@when('hue.started', 'hadoop.ready')
def check_relations(*args):
hue = Hue(get_dist_config())
hue.check_relations()
@when('hue.started', 'hive.ready')
@when_not('hive.configured')
def waiting_db(pg):
hookenv.log("%s" % (str(get_states())))
remove_state('gobinary.start')
hookenv.status_set('waiting', 'Waiting for mongodb')
def any_states(*desired_states):
"""Assert that any of the desired_states are active"""
active_states = get_states()
return any(state in active_states for state in desired_states)
def all_states(*desired_states):
"""Assert that all desired_states are active"""
active_states = get_states()
return all(state in active_states for state in desired_states)
hue.setup_hue(namenodes, resmngmrs, hdfs_port, yarn_port, yarn_http,
yarn_ipcp)
set_state('hue.configured')
@when('hue.installed', 'hadoop.ready', 'hue.configured')
@when_not('hue.started')
def start_hue(hadoop):
hookenv.status_set('maintenance', 'Setting up Hue')
hue = Hue(get_dist_config())
hue.open_ports()
hue.start()
set_state('hue.started')
if 'hue.started' in get_states():
@when_file_changed('/etc/hue/conf/hue.ini')
def restart_hue():
# Can't seem to mix @when_file_changed and @when('hue.started')
hue = Hue(get_dist_config())
hue.restart()
@when('hue.started', 'hadoop.ready')
def check_relations(*args):
hue = Hue(get_dist_config())
hue.check_relations()
@when('hue.started', 'hive.ready')