def configure_kafka(self, zk_units, network_interface=None):
# Get ip:port data from our connected zookeepers
zks = []
for unit in zk_units:
ip = utils.resolve_private_address(unit['host'])
zks.append("%s:%s" % (ip, unit['port']))
zks.sort()
zk_connect = ",".join(zks)
service, unit_num = os.environ['JUJU_UNIT_NAME'].split('/', 1)
kafka_port = self.dist_config.port('kafka')
roles = ['kafka-server']
override = {
'kafka::server::broker_id': unit_num,
'kafka::server::port': kafka_port,
'kafka::server::zookeeper_connection_string': zk_connect,
}
if network_interface:
ip = Bigtop().get_ip_for_interface(network_interface)
override['kafka::server::bind_addr'] = ip
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles, overrides=override)
bigtop.trigger_puppet()
self.set_advertise()
self.restart()
def start_datanode(namenode):
hookenv.status_set('maintenance', 'starting datanode')
# NB: service should be started by install, but we want to verify it is
# running before we set the .started state and open ports. We always
# restart here, which may seem heavy-handed. However, restart works
# whether the service is currently started or stopped. It also ensures the
# service is using the most current config.
started = host.service_restart('hadoop-hdfs-datanode')
if started:
# Create a /user/ubuntu dir in HDFS (this is safe to run multiple times).
bigtop = Bigtop()
if not bigtop.check_hdfs_setup():
try:
utils.wait_for_hdfs(30)
bigtop.setup_hdfs()
except utils.TimeoutError:
# HDFS is not yet available or is still in safe mode, so we can't
# do the initial setup (create dirs); skip setting the .started
# state below so that we try again on the next hook.
hookenv.status_set('waiting', 'waiting on hdfs')
return
# HDFS is ready. Open ports and set .started, status, and app version
for port in get_layer_opts().exposed_ports('datanode'):
hookenv.open_port(port)
set_state('apache-bigtop-datanode.started')
hookenv.status_set('maintenance', 'datanode started')
hookenv.application_version_set(get_hadoop_version())
else:
hookenv.log('DataNode failed to start')
hookenv.status_set('blocked', 'datanode failed to start')
remove_state('apache-bigtop-datanode.started')
for port in get_layer_opts().exposed_ports('datanode'):
hookenv.close_port(port)
def install_pig(self):
'''
Trigger the Bigtop puppet recipe that handles the Pig service.
'''
# Dirs are handled by the bigtop deb. No need to call out to
# dist_config to do that work.
roles = ['pig-client']
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles)
bigtop.trigger_puppet()
# Set app version for juju status output; pig --version looks like:
# Apache Pig version 0.15.0 (r: unknown)
# compiled Feb 06 2016, 23:00:40
try:
pig_out = check_output(['pig', '-x', 'local', '--version']).decode()
except CalledProcessError as e:
pig_out = e.output
lines = pig_out.splitlines()
parts = lines[0].split() if lines else []
if len(parts) < 4:
hookenv.log('Error getting Pig version: {}'.format(pig_out),
hookenv.ERROR)
pig_ver = ''
else:
pig_ver = parts[3]
hookenv.application_version_set(pig_ver)
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the namenode is ready to serve clients. Send all
dfs relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
client.send_spec(bigtop.spec())
client.send_namenodes([fqdn])
client.send_ports(hdfs_port, webhdfs_port)
# namenode.ready implies we have at least 1 datanode, which means hdfs
# is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-namenode.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map and clustername are required by the dfs interface to signify
# NN's readiness. Send it, even though they are not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
client.send_clustername(hookenv.service_name())
def send_nm_all_info(nodemanager):
"""Send nodemanagers all mapred-slave relation data.
At this point, the resourcemanager is ready to serve nodemanagers. Send all
mapred-slave relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
nodemanager.send_resourcemanagers([rm_host])
nodemanager.send_spec(bigtop.spec())
nodemanager.send_ports(rm_ipc, jh_http, jh_ipc)
# hosts_map and ssh_key are required by the mapred-slave interface to signify
# RM's readiness. Send them, even though they are not utilized by bigtop.
# NB: update KV hosts with all nodemanagers prior to sending the hosts_map
# because mapred-slave gates readiness on a NM's presence in the hosts_map.
utils.update_kv_hosts(nodemanager.hosts_map())
nodemanager.send_hosts_map(utils.get_kv_hosts())
nodemanager.send_ssh_key('invalid')
# update status with slave count and report ready for hdfs
num_slaves = len(nodemanager.nodes())
hookenv.status_set('active', 'ready ({count} nodemanager{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-resourcemanager.ready')
def install_resourcemanager(namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the RM install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing resourcemanager')
nn_host = namenode.namenodes()[0]
rm_host = get_fqdn()
bigtop = Bigtop()
hosts = {'namenode': nn_host, 'resourcemanager': rm_host}
bigtop.render_site_yaml(hosts=hosts, roles='resourcemanager')
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: mapred-slave)
# to signify RM's readiness. Set our RM info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# Add our ubuntu user to the hadoop and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-resourcemanager.installed')
hookenv.status_set('maintenance', 'resourcemanager installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def send_dn_all_info(datanode):
"""Send datanodes all dfs-slave relation data.
At this point, the namenode is ready to serve datanodes. Send all
dfs-slave relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
datanode.send_spec(bigtop.spec())
datanode.send_namenodes([fqdn])
datanode.send_ports(hdfs_port, webhdfs_port)
# hosts_map, ssh_key, and clustername are required by the dfs-slave
# interface to signify NN's readiness. Send them, even though they are not
# utilized by bigtop.
# NB: update KV hosts with all datanodes prior to sending the hosts_map
# because dfs-slave gates readiness on a DN's presence in the hosts_map.
utils.update_kv_hosts(datanode.hosts_map())
datanode.send_hosts_map(utils.get_kv_hosts())
datanode.send_ssh_key('invalid')
datanode.send_clustername(hookenv.service_name())
# update status with slave count and report ready for hdfs
num_slaves = len(datanode.nodes())
hookenv.status_set('active', 'ready ({count} datanode{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-namenode.ready')
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the namenode is ready to serve clients. Send all
dfs relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
client.send_spec(bigtop.spec())
client.send_namenodes([fqdn])
client.send_ports(hdfs_port, webhdfs_port)
# namenode.ready implies we have at least 1 datanode, which means hdfs
# is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-namenode.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map and clustername are required by the dfs interface to signify
# NN's readiness. Send it, even though they are not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
client.send_clustername(hookenv.service_name())
def send_dn_all_info(datanode):
"""Send datanodes all dfs-slave relation data.
At this point, the namenode is ready to serve datanodes. Send all
dfs-slave relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
datanode.send_spec(bigtop.spec())
datanode.send_namenodes([fqdn])
datanode.send_ports(hdfs_port, webhdfs_port)
# hosts_map, ssh_key, and clustername are required by the dfs-slave
# interface to signify NN's readiness. Send them, even though they are not
# utilized by bigtop.
# NB: update KV hosts with all datanodes prior to sending the hosts_map
# because dfs-slave gates readiness on a DN's presence in the hosts_map.
utils.update_kv_hosts(datanode.hosts_map())
datanode.send_hosts_map(utils.get_kv_hosts())
datanode.send_ssh_key('invalid')
datanode.send_clustername(hookenv.service_name())
# update status with slave count and report ready for hdfs
num_slaves = len(datanode.nodes())
hookenv.status_set(
'active', 'ready ({count} datanode{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-namenode.ready')
def install_nodemanager(namenode, resourcemanager):
"""Install if we have FQDNs.
We only need the master FQDNs to perform the nodemanager install, so poll
for master host data from the appropriate relation. This allows us to
install asap, even if '<master>.ready' is not set.
"""
namenodes = namenode.namenodes()
resourcemanagers = resourcemanager.resourcemanagers()
masters = namenodes + resourcemanagers
if namenodes and resourcemanagers and data_changed('nm.masters', masters):
installed = is_state('apache-bigtop-nodemanager.installed')
action = 'installing' if not installed else 'configuring'
hookenv.status_set('maintenance', '%s nodemanager' % action)
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': namenodes[0],
'resourcemanager': resourcemanagers[0],
},
roles=[
'nodemanager',
'mapred-app',
],
)
bigtop.queue_puppet()
set_state('apache-bigtop-nodemanager.pending')
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the resourcemanager is ready to serve clients. Send all
mapred relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
client.send_resourcemanagers([rm_host])
client.send_spec(bigtop.spec())
client.send_ports(rm_ipc, jh_http, jh_ipc)
# resourcemanager.ready implies we have at least 1 nodemanager, which means
# yarn is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-resourcemanager.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map is required by the mapred interface to signify
# RM's readiness. Send it, even though it is not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
nn_host = get_fqdn()
hosts = {'namenode': nn_host}
bigtop.render_site_yaml(hosts=hosts, roles='namenode')
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# make our namenode listen on all interfaces
hdfs_site = Path('/etc/hadoop/conf/hdfs-site.xml')
with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
props['dfs.namenode.rpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.servicerpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.http-bind-host'] = '0.0.0.0'
props['dfs.namenode.https-bind-host'] = '0.0.0.0'
# We need to create the 'mapred' user/group since we are not installing
# hadoop-mapreduce. This is needed so the namenode can access yarn
# job history files in hdfs. Also add our ubuntu user to the hadoop
# and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def test_handle_queued_puppet(self, mock_ver, mock_hookenv, mock_trigger):
'''
Verify that we attempt to call puppet when it has been queued, and
then clear the queued state.
'''
set_state('apache-bigtop-base.puppet_queued')
mock_ver.return_value = '1.2.0'
Bigtop._handle_queued_puppet()
self.assertTrue(mock_trigger.called)
self.assertFalse(is_state('apache-bigtop-base.puppet_queued'))
def test_handle_queued_puppet(self, mock_ver, mock_hookenv, mock_trigger):
'''
Verify that we attempt to call puppet when it has been queued, and
then clear the queued state.
'''
set_state('apache-bigtop-base.puppet_queued')
mock_ver.return_value = '1.2.0'
Bigtop._handle_queued_puppet()
self.assertTrue(mock_trigger.called)
self.assertFalse(is_state('apache-bigtop-base.puppet_queued'))
def install_oozie(self):
roles = ['hadoop-client']
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles)
bigtop.trigger_puppet()
roles = ['oozie-client', 'oozie-server']
bigtop.render_site_yaml(roles=roles)
bigtop.trigger_puppet()
def trigger_bigtop(self):
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
bigtop.render_site_yaml(
roles=[
'zeppelin-server',
],
overrides=overrides,
)
bigtop.trigger_puppet()
self.wait_for_api(30)
def install_hadoop_client_yarn(principal, namenode, resourcemanager):
if namenode.namenodes() and resourcemanager.resourcemanagers():
hookenv.status_set('maintenance', 'installing plugin (yarn)')
nn_host = namenode.namenodes()[0]
rm_host = resourcemanager.resourcemanagers()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_host, 'resourcemanager': rm_host}
bigtop.render_site_yaml(hosts=hosts, roles='hadoop-client')
bigtop.trigger_puppet()
set_state('apache-bigtop-plugin.yarn.installed')
hookenv.status_set('maintenance', 'plugin (yarn) installed')
else:
hookenv.status_set('waiting', 'waiting for master fqdns')
def install(self, hbase=None, zk_units=None):
'''
Trigger the Bigtop puppet recipe that handles the Hive service.
'''
# Dirs are handled by the bigtop deb. No need to call out to
# dist_config to do that. We do want 'ubuntu' in the hive group though.
self.dist_config.add_users()
# Prep config
roles = ['hive-client', 'hive-metastore', 'hive-server2']
metastore = "thrift://{}:9083".format(hookenv.unit_private_ip())
if hbase:
roles.append('hive-hbase')
hb_connect = "{}:{}".format(hbase['host'], hbase['master_port'])
zk_hbase_connect = hbase['zk_connect']
else:
hb_connect = ""
zk_hbase_connect = ""
if zk_units:
hive_support_concurrency = True
zk_hive_connect = self.get_zk_connect(zk_units)
else:
hive_support_concurrency = False
zk_hive_connect = ""
override = {
'hadoop_hive::common_config::hbase_master': hb_connect,
'hadoop_hive::common_config::hbase_zookeeper_quorum':
zk_hbase_connect,
'hadoop_hive::common_config::hive_zookeeper_quorum':
zk_hive_connect,
'hadoop_hive::common_config::hive_support_concurrency':
hive_support_concurrency,
'hadoop_hive::common_config::metastore_uris': metastore,
'hadoop_hive::common_config::server2_thrift_port':
self.dist_config.port('hive-thrift'),
'hadoop_hive::common_config::server2_thrift_http_port':
self.dist_config.port('hive-thrift-web'),
}
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles, overrides=override)
bigtop.trigger_puppet()
# Bigtop doesn't create a hive-env.sh, but we need it for heap config
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
if not hive_env.exists():
(self.dist_config.path('hive_conf') / 'hive-env.sh.template').copy(
hive_env)
def install(self, nodes=None):
'''
Write out the config, then run puppet.
After this runs, we should have a configured and running service.
'''
bigtop = Bigtop()
log("Rendering site yaml ''with overrides: {}".format(self._override))
bigtop.render_site_yaml(self._hosts, self._roles, self._override)
bigtop.trigger_puppet()
if self.is_zk_leader():
zkpeer = RelationBase.from_state('zkpeer.joined')
zkpeer.set_zk_leader()
def install_mahout():
hookenv.status_set('maintenance', 'installing mahout')
bigtop = Bigtop()
bigtop.render_site_yaml(
roles=[
'mahout-client',
],
)
bigtop.trigger_puppet()
with utils.environment_edit_in_place('/etc/environment') as env:
env['MAHOUT_HOME'] = '/usr/lib/mahout'
hookenv.status_set('active', 'ready')
set_state('mahout.installed')
def install_oozie(self):
roles = ['hadoop-client']
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles)
bigtop.trigger_puppet()
roles = ['oozie-client', 'oozie-server']
bigtop.render_site_yaml(roles=roles)
bigtop.trigger_puppet()
def _override(self):
'''
Return a dict of keys and values that will override puppet's
defaults.
'''
override = {
"hadoop_zookeeper::server::myid": local_unit().split("/")[1],
"hadoop_zookeeper::server::ensemble": self.read_peers()
}
conf = config()
network_interface = conf.get('network_interface')
autopurge_purge_interval = conf.get('autopurge_purge_interval')
autopurge_snap_retain_count = conf.get('autopurge_snap_retain_count')
if network_interface:
key = "hadoop_zookeeper::server::client_bind_addr"
override[key] = Bigtop().get_ip_for_interface(network_interface)
if autopurge_purge_interval:
key = "hadoop_zookeeper::server::autopurge_purge_interval"
override[key] = autopurge_purge_interval
if autopurge_snap_retain_count:
key = "hadoop_zookeeper::server::autopurge_snap_retain_count"
override[key] = autopurge_snap_retain_count
return override
def install_mahout():
hookenv.status_set('maintenance', 'installing mahout')
bigtop = Bigtop()
bigtop.render_site_yaml(
roles=[
'mahout-client',
],
)
bigtop.trigger_puppet()
with utils.environment_edit_in_place('/etc/environment') as env:
env['MAHOUT_HOME'] = '/usr/lib/mahout'
set_state('mahout.installed')
hookenv.status_set('active', 'ready')
# set app version string for juju status output
mahout_version = get_package_version('mahout') or 'unknown'
hookenv.application_version_set(mahout_version)
def configure_kafka(self, zk_units, network_interface=None):
# Get ip:port data from our connected zookeepers
zks = []
for unit in zk_units:
ip = utils.resolve_private_address(unit['host'])
zks.append("%s:%s" % (ip, unit['port']))
zks.sort()
zk_connect = ",".join(zks)
service, unit_num = os.environ['JUJU_UNIT_NAME'].split('/', 1)
kafka_port = self.dist_config.port('kafka')
roles = ['kafka-server']
override = {
'kafka::server::broker_id': unit_num,
'kafka::server::port': kafka_port,
'kafka::server::zookeeper_connection_string': zk_connect,
}
if network_interface:
ip = Bigtop().get_ip_for_interface(network_interface)
override['kafka::server::bind_addr'] = ip
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles, overrides=override)
bigtop.trigger_puppet()
self.set_advertise()
self.restart()
def install_hadoop_client_hdfs(principal, namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the plugin install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing plugin (hdfs)')
nn_host = namenode.namenodes()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_host}
bigtop.render_site_yaml(hosts=hosts, roles='hadoop-client')
bigtop.trigger_puppet()
set_state('apache-bigtop-plugin.hdfs.installed')
hookenv.status_set('maintenance', 'plugin (hdfs) installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def configure(self, hosts, zk_units):
zks = []
for unit in zk_units:
ip = utils.resolve_private_address(unit["host"])
zks.append(ip)
zks.sort()
zk_connect = ",".join(zks)
roles = ["hbase-server", "hbase-master", "hbase-client"]
override = {
"hadoop_hbase::common_config::zookeeper_quorum": zk_connect,
"hadoop_hbase::deploy::auxiliary": False,
}
bigtop = Bigtop()
bigtop.render_site_yaml(hosts, roles, override)
bigtop.trigger_puppet()
def install_giraph(giraph):
"""Install giraph when prerequisite states are present."""
hookenv.status_set('maintenance', 'installing giraph')
bigtop = Bigtop()
bigtop.render_site_yaml(
roles=[
'giraph-client',
],
)
bigtop.trigger_puppet()
# Put down the -doc subpackage so we get giraph-examples
fetch.apt_install('giraph-doc')
giraph_home = Path('/usr/lib/giraph')
giraph_docdir = Path('/usr/share/doc/giraph')
giraph_libdir = Path(giraph_home / 'lib')
giraph_examples = glob('{}/giraph-examples-*.jar'.format(giraph_docdir))
# Gather a list of all the giraph jars (needed for -libjars)
giraph_jars = giraph_examples
giraph_jars.extend(get_good_jars(giraph_home, prefix=True))
giraph_jars.extend(get_good_jars(giraph_libdir, prefix=True))
# Update environment with appropriate giraph bits. HADOOP_CLASSPATH can
# use wildcards (and it should for readability), but GIRAPH_JARS, which
# is intended to be used as 'hadoop jar -libjars $GIRAPH_JARS', needs to
# be a comma-separate list of jars.
with utils.environment_edit_in_place('/etc/environment') as env:
cur_cp = env['HADOOP_CLASSPATH'] if 'HADOOP_CLASSPATH' in env else ""
env['GIRAPH_HOME'] = giraph_home
env['HADOOP_CLASSPATH'] = "{examples}/*:{home}/*:{libs}/*:{cp}".format(
examples=giraph_docdir,
home=giraph_home,
libs=giraph_libdir,
cp=cur_cp
)
env['GIRAPH_JARS'] = ','.join(j for j in giraph_jars)
set_state('giraph.installed')
report_status()
# set app version string for juju status output
giraph_version = get_package_version('giraph') or 'unknown'
hookenv.application_version_set(giraph_version)
def install_resourcemanager(namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the RM install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing resourcemanager')
nn_host = namenode.namenodes()[0]
rm_host = get_fqdn()
bigtop = Bigtop()
hosts = {'namenode': nn_host, 'resourcemanager': rm_host}
bigtop.render_site_yaml(hosts=hosts, roles='resourcemanager')
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: mapred-slave)
# to signify RM's readiness. Set our RM info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# Add our ubuntu user to the hadoop and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-resourcemanager.installed')
hookenv.status_set('maintenance', 'resourcemanager installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def install_component(namenode, resourcemanager):
'''Install only if nn and rm have sent their FQDNs.'''
if namenode.namenodes() and resourcemanager.resourcemanagers():
set_state('components.installed')
options = layer.options('apache-bigtop-base')
components = options.get("bigtop_component_list")
hookenv.status_set('maintenance', 'installing {}'.format(components))
nn_fqdn = namenode.namenodes()[0]
rm_fqdn = resourcemanager.resourcemanagers()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_fqdn, 'resourcemanager': rm_fqdn}
bigtop.install_component(RM=rm_fqdn, NN=nn_fqdn)
for component in components.split():
try:
for port in get_layer_opts().exposed_ports(component):
hookenv.open_port(port)
except AttributeError:
hookenv.log("Not opening ports for component {}".format(component))
hookenv.status_set('active', 'ready')
else:
hookenv.status_set('waiting',
'waiting for namenode and resource manager to become ready')
def configure(self, available_hosts):
"""
This is the core logic of setting up spark.
Two flags are needed:
* Namenode exists aka HDFS is there
* Resource manager exists aka YARN is ready
both flags are infered from the available hosts.
:param dict available_hosts: Hosts that Spark should know about.
"""
if not unitdata.kv().get('spark.bootstrapped', False):
self.setup()
unitdata.kv().set('spark.bootstrapped', True)
self.install_benchmark()
hosts = {
'spark': available_hosts['spark-master'],
}
dc = self.dist_config
events_log_dir = 'file://{}'.format(dc.path('spark_events'))
if 'namenode' in available_hosts:
hosts['namenode'] = available_hosts['namenode']
events_log_dir = self.setup_hdfs_logs()
if 'resourcemanager' in available_hosts:
hosts['resourcemanager'] = available_hosts['resourcemanager']
roles = self.get_roles()
override = {
'spark::common::master_url': self.get_master_url(available_hosts['spark-master']),
'spark::common::event_log_dir': events_log_dir,
'spark::common::history_log_dir': events_log_dir,
}
bigtop = Bigtop()
bigtop.render_site_yaml(hosts, roles, override)
bigtop.trigger_puppet()
# There is a race condition here.
# The work role will not start the first time we trigger puppet apply.
# The exception in /var/logs/spark:
# Exception in thread "main" org.apache.spark.SparkException: Invalid master URL: spark://:7077
# The master url is not set at the time the worker start the first time.
# TODO(kjackal): ...do the needed... (investiate,debug,submit patch)
bigtop.trigger_puppet()
if 'namenode' not in available_hosts:
# Make sure users other than spark can access the events logs dir and run jobs
utils.run_as('root', 'chmod', '777', dc.path('spark_events'))
def fetch_bigtop():
hookenv.status_set('maintenance', 'installing apache bigtop base')
try:
Bigtop().install()
except UnhandledSource as e:
hookenv.status_set('blocked', 'unable to fetch apache bigtop: %s' % e)
except ChecksumError:
hookenv.status_set(
'waiting', 'unable to fetch apache bigtop: checksum error'
' (will retry)')
else:
hookenv.status_set('waiting', 'apache bigtop base installed')
set_state('bigtop.available')
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
bigtop.render_site_yaml(
hosts={
'namenode': get_fqdn(),
},
roles=[
'namenode',
'mapred-app',
],
# NB: We want the NN to listen on all interfaces, so bind to 0.0.0.0.
overrides={
'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port,
'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port,
'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0',
}
)
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# We need to create the 'mapred' and 'spark' user/group since we may not
# be installing hadoop-mapreduce or spark on this machine. This is needed
# so the namenode can access yarn and spark job history files in hdfs. Also
# add our ubuntu user to the hadoop, mapred, and spark groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def trigger_bigtop(self):
'''
Trigger the Bigtop puppet recipe that handles the Zeppelin service.
'''
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
# The zep deb depends on spark-core which unfortunately brings in
# most of hadoop. Include appropriate roles here to ensure these
# packages are configured in the same way as our other Bigtop
# software deployed with puppet.
bigtop.render_site_yaml(
roles=[
'spark-client',
'spark-yarn-slave',
'zeppelin-server',
],
overrides=overrides,
)
bigtop.trigger_puppet()
self.wait_for_api(30)
def trigger_bigtop(self):
'''
Trigger the Bigtop puppet recipe that handles the Zeppelin service.
'''
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
# The zep deb depends on spark-core which unfortunately brings in
# most of hadoop. Include appropriate roles here to ensure these
# packages are configured in the same way as our other Bigtop
# software deployed with puppet.
bigtop.render_site_yaml(
roles=[
'spark-client',
'spark-yarn-slave',
'zeppelin-server',
],
overrides=overrides,
)
# NB: during an upgrade, we configure the site.yaml, but do not
# trigger puppet. The user must do that with the 'reinstall' action.
if unitdata.kv().get('zeppelin.version.repo', False):
hookenv.log("An upgrade is available and the site.yaml has been "
"configured. Run the 'reinstall' action to continue.",
level=hookenv.INFO)
else:
####################################################################
# BUG: BIGTOP-2742
# Default zeppelin init script looks for the literal '$(hostname)'
# string. Symlink it so it exists before the apt install from puppet
# tries to start the service.
import subprocess
host = subprocess.check_output(['hostname']).decode('utf8').strip()
zepp_pid = '/var/run/zeppelin/zeppelin-zeppelin-{}.pid'.format(host)
utils.run_as('root', 'mkdir', '-p', '/var/run/zeppelin')
utils.run_as('root', 'ln', '-sf',
zepp_pid,
'/var/run/zeppelin/zeppelin-zeppelin-$(hostname).pid')
####################################################################
bigtop.trigger_puppet()
self.wait_for_api(30)
####################################################################
# BUG: BIGTOP-2742
# Puppet apply will call systemctl daemon-reload, which removes the
# symlink we just created. Now that the bits are on disk, update the
# init script $(hostname) that caused this mess to begin with.
zepp_init_script = '/etc/init.d/zeppelin'
utils.re_edit_in_place(zepp_init_script, {
r'^# pidfile.*': '# pidfile: {}'.format(zepp_pid),
})
utils.run_as('root', 'systemctl', 'daemon-reload')
self.restart()
self.wait_for_api(30)
def install_component(namenode, resourcemanager):
'''Install only if nn and rm have sent their FQDNs.'''
if namenode.namenodes() and resourcemanager.resourcemanagers():
set_state('components.installed')
options = layer.options('apache-bigtop-base')
components = options.get("bigtop_component_list")
hookenv.status_set('maintenance', 'installing {}'.format(components))
nn_fqdn = namenode.namenodes()[0]
rm_fqdn = resourcemanager.resourcemanagers()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_fqdn, 'resourcemanager': rm_fqdn}
bigtop.install_component(RM=rm_fqdn, NN=nn_fqdn)
for component in components.split():
try:
for port in get_layer_opts().exposed_ports(component):
hookenv.open_port(port)
except AttributeError:
hookenv.log(
"Not opening ports for component {}".format(component))
hookenv.status_set('active', 'ready')
else:
hookenv.status_set(
'waiting',
'waiting for namenode and resource manager to become ready')
def install_datanode(namenode):
"""
Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the datanode install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
namenodes = namenode.namenodes()
if namenodes and data_changed('datanode.namenodes', namenodes):
installed = is_state('apache-bigtop-datanode.installed')
action = 'installing' if not installed else 'configuring'
hookenv.status_set('maintenance', '%s datanode' % action)
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': namenodes[0],
},
roles=[
'datanode',
],
)
bigtop.queue_puppet()
set_state('apache-bigtop-datanode.pending')
def _override(self):
'''
Return a dict of keys and values that will override puppet's
defaults.
'''
override = {
"hadoop_zookeeper::server::myid": local_unit().split("/")[1],
"hadoop_zookeeper::server::ensemble": self.read_peers()
}
network_interface = config().get('network_interface')
if network_interface:
key = "hadoop_zookeeper::server::client_bind_addr"
override[key] = Bigtop().get_ip_for_interface(network_interface)
return override
def check_repo_version():
"""
Configure a bigtop site.yaml if a new version of spark is available.
This method will set unitdata if a different version of spark-core is
available in the newly configured bigtop repo. This unitdata allows us to
configure site.yaml while gating the actual puppet apply. The user must do
the puppet apply by calling the 'reinstall' action.
"""
repo_ver = Bigtop().check_bigtop_repo_package('spark-core')
if repo_ver:
unitdata.kv().set('spark.version.repo', repo_ver)
unitdata.kv().flush(True)
reinstall_spark(force=True)
else:
unitdata.kv().unset('spark.version.repo')
report_status()
def install_mahout():
hookenv.status_set('maintenance', 'installing mahout')
bigtop = Bigtop()
bigtop.render_site_yaml(roles=[
'mahout-client',
], )
bigtop.trigger_puppet()
with utils.environment_edit_in_place('/etc/environment') as env:
env['MAHOUT_HOME'] = '/usr/lib/mahout'
hookenv.status_set('active', 'ready')
set_state('mahout.installed')
def install(self, hbase=None, zk_units=None):
'''
Trigger the Bigtop puppet recipe that handles the Hive service.
'''
# Dirs are handled by the bigtop deb. No need to call out to
# dist_config to do that. We do want 'ubuntu' in the hive group though.
self.dist_config.add_users()
# Prep config
roles = ['hive-client', 'hive-metastore', 'hive-server2']
metastore = "thrift://{}:9083".format(hookenv.unit_private_ip())
if hbase:
roles.append('hive-hbase')
hb_connect = "{}:{}".format(hbase['host'], hbase['master_port'])
zk_hbase_connect = hbase['zk_connect']
else:
hb_connect = ""
zk_hbase_connect = ""
if zk_units:
hive_support_concurrency = True
zk_hive_connect = self.get_zk_connect(zk_units)
else:
hive_support_concurrency = False
zk_hive_connect = ""
override = {
'hadoop_hive::common_config::hbase_master':
hb_connect,
'hadoop_hive::common_config::hbase_zookeeper_quorum':
zk_hbase_connect,
'hadoop_hive::common_config::hive_zookeeper_quorum':
zk_hive_connect,
'hadoop_hive::common_config::hive_support_concurrency':
hive_support_concurrency,
'hadoop_hive::common_config::metastore_uris':
metastore,
'hadoop_hive::common_config::server2_thrift_port':
self.dist_config.port('hive-thrift'),
'hadoop_hive::common_config::server2_thrift_http_port':
self.dist_config.port('hive-thrift-web'),
}
bigtop = Bigtop()
bigtop.render_site_yaml(roles=roles, overrides=override)
bigtop.trigger_puppet()
# Bigtop doesn't create a hive-env.sh, but we need it for heap config
hive_env = self.dist_config.path('hive_conf') / 'hive-env.sh'
if not hive_env.exists():
(self.dist_config.path('hive_conf') /
'hive-env.sh.template').copy(hive_env)
def trigger_bigtop(self):
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
bigtop.render_site_yaml(
roles=[
'zeppelin-server',
],
overrides=overrides,
)
bigtop.trigger_puppet()
self.wait_for_api(30)
def check_repo_version():
"""
Configure a bigtop site.yaml if a new version of zeppelin is available.
This method will set unitdata if a different version of zeppelin is
available in the newly configured bigtop repo. This unitdata allows us to
configure site.yaml while gating the actual puppet apply. The user must do
the puppet apply by calling the 'reinstall' action.
"""
repo_ver = Bigtop().check_bigtop_repo_package('zeppelin')
if repo_ver:
unitdata.kv().set('zeppelin.version.repo', repo_ver)
unitdata.kv().flush(True)
zeppelin = Zeppelin()
zeppelin.trigger_bigtop()
else:
unitdata.kv().unset('zeppelin.version.repo')
update_status()
def install_hadoop_client_yarn(principal, namenode, resourcemanager):
if namenode.namenodes() and resourcemanager.resourcemanagers():
hookenv.status_set('maintenance', 'installing plugin (yarn)')
nn_host = namenode.namenodes()[0]
rm_host = resourcemanager.resourcemanagers()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_host, 'resourcemanager': rm_host}
bigtop.render_site_yaml(hosts=hosts, roles='hadoop-client')
bigtop.trigger_puppet()
set_state('apache-bigtop-plugin.yarn.installed')
hookenv.status_set('maintenance', 'plugin (yarn) installed')
else:
hookenv.status_set('waiting', 'waiting for master fqdns')
def change_bigtop_version():
"""
Update Bigtop source if a new version has been requested.
This method will set the 'bigtop.version.changed' state that can be used
by charms to check for potential upgrades.
"""
# Make sure the config has actually changed; on initial initial,
# config.changed.foo is always set. Check for a previous value that is
# different than our current value before fetching new source.
cur_ver = hookenv.config()['bigtop_version']
pre_ver = hookenv.config().previous('bigtop_version')
if pre_ver and cur_ver != pre_ver:
hookenv.log(
'Bigtop version {} requested over {}. Refreshing source.'.format(
cur_ver, pre_ver))
Bigtop().refresh_bigtop_release()
hookenv.status_set('waiting', 'pending scan for package updates')
set_state('bigtop.version.changed')
def configure(self, hosts, zk_units):
zk_connect = self.get_zk_connect(zk_units)
roles = ['hbase-server', 'hbase-master', 'hbase-client']
override = {
'bigtop::hbase_thrift_port': self.dist_config.port('hbase-thrift'),
'hadoop_hbase::client::thrift': True,
'hadoop_hbase::common_config::heap_size': hookenv.config()['heap'],
'hadoop_hbase::common_config::zookeeper_quorum': zk_connect,
'hadoop_hbase::deploy::auxiliary': False,
}
bigtop = Bigtop()
bigtop.render_site_yaml(hosts, roles, override)
bigtop.trigger_puppet()
def install(self, nodes=None):
'''
Write out the config, then run puppet.
After this runs, we should have a configured and running service.
'''
bigtop = Bigtop()
log("Rendering site yaml ''with overrides: {}".format(self._override))
bigtop.render_site_yaml(self._hosts, self._roles, self._override)
bigtop.trigger_puppet()
if self.is_zk_leader():
zkpeer = RelationBase.from_state('zkpeer.joined')
zkpeer.set_zk_leader()
def install_mahout():
hookenv.status_set('maintenance', 'installing mahout')
bigtop = Bigtop()
bigtop.render_site_yaml(roles=[
'mahout-client',
], )
bigtop.trigger_puppet()
with utils.environment_edit_in_place('/etc/environment') as env:
env['MAHOUT_HOME'] = '/usr/lib/mahout'
set_state('mahout.installed')
hookenv.status_set('active', 'ready')
# set app version string for juju status output
mahout_version = get_package_version('mahout') or 'unknown'
hookenv.application_version_set(mahout_version)
def trigger_bigtop(self):
'''
Trigger the Bigtop puppet recipe that handles the Zeppelin service.
'''
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
bigtop.render_site_yaml(
roles=[
'zeppelin-server',
],
overrides=overrides,
)
bigtop.trigger_puppet()
self.wait_for_api(30)
def install_hadoop_client_hdfs(principal, namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the plugin install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing plugin (hdfs)')
nn_host = namenode.namenodes()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_host}
bigtop.render_site_yaml(hosts=hosts, roles='hadoop-client')
bigtop.trigger_puppet()
set_state('apache-bigtop-plugin.hdfs.installed')
hookenv.status_set('maintenance', 'plugin (hdfs) installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def configure(self, hosts, zk_units):
zks = []
for unit in zk_units:
ip = utils.resolve_private_address(unit['host'])
zks.append(ip)
zks.sort()
zk_connect = ",".join(zks)
roles = ['hbase-server', 'hbase-master', 'hbase-client']
override = {
'hadoop_hbase::common_config::zookeeper_quorum': zk_connect,
'hadoop_hbase::deploy::auxiliary': False
}
bigtop = Bigtop()
bigtop.render_site_yaml(hosts, roles, override)
bigtop.trigger_puppet()
def install_giraph(giraph):
"""Install giraph when prerequisite states are present."""
hookenv.status_set('maintenance', 'installing giraph')
bigtop = Bigtop()
bigtop.render_site_yaml(roles=[
'giraph-client',
], )
bigtop.trigger_puppet()
# Put down the -doc subpackage so we get giraph-examples
fetch.apt_install('giraph-doc')
giraph_home = Path('/usr/lib/giraph')
giraph_docdir = Path('/usr/share/doc/giraph')
giraph_libdir = Path(giraph_home / 'lib')
giraph_examples = glob('{}/giraph-examples-*.jar'.format(giraph_docdir))
# Gather a list of all the giraph jars (needed for -libjars)
giraph_jars = giraph_examples
giraph_jars.extend(get_good_jars(giraph_home, prefix=True))
giraph_jars.extend(get_good_jars(giraph_libdir, prefix=True))
# Update environment with appropriate giraph bits. HADOOP_CLASSPATH can
# use wildcards (and it should for readability), but GIRAPH_JARS, which
# is intended to be used as 'hadoop jar -libjars $GIRAPH_JARS', needs to
# be a comma-separate list of jars.
with utils.environment_edit_in_place('/etc/environment') as env:
cur_cp = env['HADOOP_CLASSPATH'] if 'HADOOP_CLASSPATH' in env else ""
env['GIRAPH_HOME'] = giraph_home
env['HADOOP_CLASSPATH'] = "{examples}/*:{home}/*:{libs}/*:{cp}".format(
examples=giraph_docdir,
home=giraph_home,
libs=giraph_libdir,
cp=cur_cp)
env['GIRAPH_JARS'] = ','.join(j for j in giraph_jars)
set_state('giraph.installed')
report_status()
# set app version string for juju status output
giraph_version = get_package_version('giraph') or 'unknown'
hookenv.application_version_set(giraph_version)
def trigger_bigtop(self):
'''
Trigger the Bigtop puppet recipe that handles the Zeppelin service.
'''
bigtop = Bigtop()
overrides = unitdata.kv().getrange('zeppelin.bigtop.overrides.',
strip=True)
# The zep deb depends on spark-core which unfortunately brings in
# most of hadoop. Include appropriate roles here to ensure these
# packages are configured in the same way as our other Bigtop
# software deployed with puppet.
bigtop.render_site_yaml(
roles=[
'spark-client',
'spark-yarn-slave',
'zeppelin-server',
],
overrides=overrides,
)
bigtop.trigger_puppet()
self.wait_for_api(30)
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
bigtop.render_site_yaml(
hosts={
'namenode': get_fqdn(),
},
roles=[
'namenode',
'mapred-app',
],
# NB: We want the NN to listen on all interfaces, so bind to 0.0.0.0.
overrides={
'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port,
'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port,
'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0',
})
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# We need to create the 'mapred' user/group since we are not installing
# hadoop-mapreduce. This is needed so the namenode can access yarn
# job history files in hdfs. Also add our ubuntu user to the hadoop
# and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': get_fqdn(),
},
roles=[
'namenode',
'mapred-app',
],
)
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# make our namenode listen on all interfaces
hdfs_site = Path('/etc/hadoop/conf/hdfs-site.xml')
with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
props['dfs.namenode.rpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.servicerpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.http-bind-host'] = '0.0.0.0'
props['dfs.namenode.https-bind-host'] = '0.0.0.0'
# We need to create the 'mapred' user/group since we are not installing
# hadoop-mapreduce. This is needed so the namenode can access yarn
# job history files in hdfs. Also add our ubuntu user to the hadoop
# and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def install_resourcemanager(namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the RM install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing resourcemanager')
# Hosts
nn_host = namenode.namenodes()[0]
rm_host = get_fqdn()
# Ports
rm_ipc = get_layer_opts().port('resourcemanager')
rm_http = get_layer_opts().port('rm_webapp_http')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
hdfs_port = namenode.port()
webhdfs_port = namenode.webhdfs_port()
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': nn_host,
'resourcemanager': rm_host,
},
roles=[
'resourcemanager',
],
# NB: When we colocate the NN and RM, the RM will run puppet apply
# last. To ensure we don't lose any hdfs-site.xml data set by the
# NN, override common_hdfs properties again here.
overrides={
'hadoop::common_yarn::hadoop_rm_port': rm_ipc,
'hadoop::common_yarn::hadoop_rm_webapp_port': rm_http,
'hadoop::common_yarn::hadoop_rm_bind_host': '0.0.0.0',
'hadoop::common_mapred_app::mapreduce_jobhistory_host': '0.0.0.0',
'hadoop::common_mapred_app::mapreduce_jobhistory_port': jh_ipc,
'hadoop::common_mapred_app::mapreduce_jobhistory_webapp_port': jh_http,
'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port,
'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port,
'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0',
}
)
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: mapred-slave)
# to signify RM's readiness. Set our RM info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# We need to create the 'spark' user/group since we may not be
# installing spark on this machine. This is needed so the history
# server can access spark job history files in hdfs. Also add our
# ubuntu user to the hadoop, mapred, and spark groups on this machine.
get_layer_opts().add_users()
set_state('apache-bigtop-resourcemanager.installed')
hookenv.status_set('maintenance', 'resourcemanager installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def send_nn_spec(namenode):
"""Send our resourcemanager spec so the namenode can become ready."""
bigtop = Bigtop()
namenode.set_local_spec(bigtop.spec())
hookenv.status_set('waiting', 'waiting for namenode to become ready')
def send_nm_spec(namenode, resourcemanager):
"""Send our nodemanager spec so the master relations can become ready."""
bigtop = Bigtop()
namenode.set_local_spec(bigtop.spec())
resourcemanager.set_local_spec(bigtop.spec())
