def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the namenode is ready to serve clients. Send all
dfs relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
client.send_spec(bigtop.spec())
client.send_namenodes([fqdn])
client.send_ports(hdfs_port, webhdfs_port)
# namenode.ready implies we have at least 1 datanode, which means hdfs
# is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-namenode.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map and clustername are required by the dfs interface to signify
# NN's readiness. Send it, even though they are not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
client.send_clustername(hookenv.service_name())
def start_datanode(namenode):
hookenv.status_set('maintenance', 'starting datanode')
# NB: service should be started by install, but we want to verify it is
# running before we set the .started state and open ports. We always
# restart here, which may seem heavy-handed. However, restart works
# whether the service is currently started or stopped. It also ensures the
# service is using the most current config.
started = host.service_restart('hadoop-hdfs-datanode')
if started:
# Create a /user/ubuntu dir in HDFS (this is safe to run multiple times).
bigtop = Bigtop()
if not bigtop.check_hdfs_setup():
try:
utils.wait_for_hdfs(30)
bigtop.setup_hdfs()
except utils.TimeoutError:
# HDFS is not yet available or is still in safe mode, so we can't
# do the initial setup (create dirs); skip setting the .started
# state below so that we try again on the next hook.
hookenv.status_set('waiting', 'waiting on hdfs')
return
# HDFS is ready. Open ports and set .started, status, and app version
for port in get_layer_opts().exposed_ports('datanode'):
hookenv.open_port(port)
set_state('apache-bigtop-datanode.started')
hookenv.status_set('maintenance', 'datanode started')
hookenv.application_version_set(get_hadoop_version())
else:
hookenv.log('DataNode failed to start')
hookenv.status_set('blocked', 'datanode failed to start')
remove_state('apache-bigtop-datanode.started')
for port in get_layer_opts().exposed_ports('datanode'):
hookenv.close_port(port)
def send_dn_all_info(datanode):
"""Send datanodes all dfs-slave relation data.
At this point, the namenode is ready to serve datanodes. Send all
dfs-slave relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
datanode.send_spec(bigtop.spec())
datanode.send_namenodes([fqdn])
datanode.send_ports(hdfs_port, webhdfs_port)
# hosts_map, ssh_key, and clustername are required by the dfs-slave
# interface to signify NN's readiness. Send them, even though they are not
# utilized by bigtop.
# NB: update KV hosts with all datanodes prior to sending the hosts_map
# because dfs-slave gates readiness on a DN's presence in the hosts_map.
utils.update_kv_hosts(datanode.hosts_map())
datanode.send_hosts_map(utils.get_kv_hosts())
datanode.send_ssh_key('invalid')
datanode.send_clustername(hookenv.service_name())
# update status with slave count and report ready for hdfs
num_slaves = len(datanode.nodes())
hookenv.status_set('active', 'ready ({count} datanode{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-namenode.ready')
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the namenode is ready to serve clients. Send all
dfs relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
client.send_spec(bigtop.spec())
client.send_namenodes([fqdn])
client.send_ports(hdfs_port, webhdfs_port)
# namenode.ready implies we have at least 1 datanode, which means hdfs
# is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-namenode.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map and clustername are required by the dfs interface to signify
# NN's readiness. Send it, even though they are not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
client.send_clustername(hookenv.service_name())
def send_dn_all_info(datanode):
"""Send datanodes all dfs-slave relation data.
At this point, the namenode is ready to serve datanodes. Send all
dfs-slave relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
datanode.send_spec(bigtop.spec())
datanode.send_namenodes([fqdn])
datanode.send_ports(hdfs_port, webhdfs_port)
# hosts_map, ssh_key, and clustername are required by the dfs-slave
# interface to signify NN's readiness. Send them, even though they are not
# utilized by bigtop.
# NB: update KV hosts with all datanodes prior to sending the hosts_map
# because dfs-slave gates readiness on a DN's presence in the hosts_map.
utils.update_kv_hosts(datanode.hosts_map())
datanode.send_hosts_map(utils.get_kv_hosts())
datanode.send_ssh_key('invalid')
datanode.send_clustername(hookenv.service_name())
# update status with slave count and report ready for hdfs
num_slaves = len(datanode.nodes())
hookenv.status_set(
'active', 'ready ({count} datanode{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-namenode.ready')
def install_resourcemanager(namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the RM install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing resourcemanager')
nn_host = namenode.namenodes()[0]
rm_host = get_fqdn()
bigtop = Bigtop()
hosts = {'namenode': nn_host, 'resourcemanager': rm_host}
bigtop.render_site_yaml(hosts=hosts, roles='resourcemanager')
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: mapred-slave)
# to signify RM's readiness. Set our RM info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# Add our ubuntu user to the hadoop and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-resourcemanager.installed')
hookenv.status_set('maintenance', 'resourcemanager installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def install_resourcemanager(namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the RM install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing resourcemanager')
nn_host = namenode.namenodes()[0]
rm_host = get_fqdn()
bigtop = Bigtop()
hosts = {'namenode': nn_host, 'resourcemanager': rm_host}
bigtop.render_site_yaml(hosts=hosts, roles='resourcemanager')
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: mapred-slave)
# to signify RM's readiness. Set our RM info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# Add our ubuntu user to the hadoop and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-resourcemanager.installed')
hookenv.status_set('maintenance', 'resourcemanager installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
nn_host = get_fqdn()
hosts = {'namenode': nn_host}
bigtop.render_site_yaml(hosts=hosts, roles='namenode')
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# make our namenode listen on all interfaces
hdfs_site = Path('/etc/hadoop/conf/hdfs-site.xml')
with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
props['dfs.namenode.rpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.servicerpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.http-bind-host'] = '0.0.0.0'
props['dfs.namenode.https-bind-host'] = '0.0.0.0'
# We need to create the 'mapred' user/group since we are not installing
# hadoop-mapreduce. This is needed so the namenode can access yarn
# job history files in hdfs. Also add our ubuntu user to the hadoop
# and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def send_nm_all_info(nodemanager):
"""Send nodemanagers all mapred-slave relation data.
At this point, the resourcemanager is ready to serve nodemanagers. Send all
mapred-slave relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
nodemanager.send_resourcemanagers([rm_host])
nodemanager.send_spec(bigtop.spec())
nodemanager.send_ports(rm_ipc, jh_http, jh_ipc)
# hosts_map and ssh_key are required by the mapred-slave interface to signify
# RM's readiness. Send them, even though they are not utilized by bigtop.
# NB: update KV hosts with all nodemanagers prior to sending the hosts_map
# because mapred-slave gates readiness on a NM's presence in the hosts_map.
utils.update_kv_hosts(nodemanager.hosts_map())
nodemanager.send_hosts_map(utils.get_kv_hosts())
nodemanager.send_ssh_key('invalid')
# update status with slave count and report ready for hdfs
num_slaves = len(nodemanager.nodes())
hookenv.status_set('active', 'ready ({count} nodemanager{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-resourcemanager.ready')
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the resourcemanager is ready to serve clients. Send all
mapred relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
client.send_resourcemanagers([rm_host])
client.send_spec(bigtop.spec())
client.send_ports(rm_ipc, jh_http, jh_ipc)
# resourcemanager.ready implies we have at least 1 nodemanager, which means
# yarn is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-resourcemanager.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map is required by the mapred interface to signify
# RM's readiness. Send it, even though it is not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
def send_early_install_info(remote):
"""Send clients/slaves enough relation data to start their install.
If slaves or clients join before the namenode is installed, we can still provide enough
info to start their installation. This will help parallelize installation among our
cluster.
Note that slaves can safely install early, but should not start until the
'namenode.ready' state is set by the dfs-slave interface.
"""
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
remote.send_namenodes([fqdn])
remote.send_ports(hdfs_port, webhdfs_port)
def serve_client(client):
'''
Inform clients when hive is ready to serve.
'''
port = get_layer_opts().port('hive-thrift')
client.send_port(port)
client.set_ready()
def start_nodemanager():
hookenv.status_set('maintenance', 'starting nodemanager')
host.service_start('hadoop-yarn-nodemanager')
for port in get_layer_opts().exposed_ports('nodemanager'):
hookenv.open_port(port)
set_state('nodemanager.started')
hookenv.status_set('active', 'ready')
def send_early_install_info(remote):
"""Send clients/slaves enough relation data to start their install.
If slaves or clients join before the namenode is installed, we can still provide enough
info to start their installation. This will help parallelize installation among our
cluster.
Note that slaves can safely install early, but should not start until the
'namenode.ready' state is set by the dfs-slave interface.
"""
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
remote.send_namenodes([fqdn])
remote.send_ports(hdfs_port, webhdfs_port)
def serve_client(zk, client):
'''
We may have multiple HBase peers, but we only need to send 1 set of
connection data. Leverage Juju leadership to only send the leader
info (even if it's not the actual HBase master).
Zookeeper will ensure that any HBase peer routes requests to the
appropriate master.
'''
hbase = HBase()
# Get hbase config and zk info
config = get_layer_opts()
host = hookenv.unit_private_ip()
master_port = config.port('hbase-master')
regionserver_port = config.port('hbase-region')
thrift_port = config.port('hbase-thrift')
zk_connect = hbase.get_zk_connect(zk.zookeepers())
# Send data to our connected client
client.send_connection(master_port=master_port,
regionserver_port=regionserver_port,
thrift_port=thrift_port,
host=host,
zk_connect=zk_connect)
hookenv.log('Serving HBase client with master {}:{}, regionserver '
'port {}, thrift port {}, and zk connect {}'.format(
host, master_port, regionserver_port, thrift_port,
zk_connect))
def send_early_install_info(remote):
"""Send clients/slaves enough relation data to start their install.
If slaves or clients join before the resourcemanager is installed, we can
still provide enough info to start their installation. This will help
parallelize installation among our cluster.
Note that slaves can safely install early, but should not start until the
'resourcemanager.ready' state is set by the mapred-slave interface.
"""
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
remote.send_resourcemanagers([rm_host])
remote.send_ports(rm_ipc, jh_http, jh_ipc)
def start_namenode():
hookenv.status_set('maintenance', 'starting namenode')
# NB: service should be started by install, but this may be handy in case
# we have something that removes the .started state in the future. Also
# note we restart here in case we modify conf between install and now.
host.service_restart('hadoop-hdfs-namenode')
for port in get_layer_opts().exposed_ports('namenode'):
hookenv.open_port(port)
set_state('apache-bigtop-namenode.started')
hookenv.status_set('maintenance', 'namenode started')
def start_resourcemanager(namenode):
hookenv.status_set('maintenance', 'starting resourcemanager')
# NB: service should be started by install, but this may be handy in case
# we have something that removes the .started state in the future. Also
# note we restart here in case we modify conf between install and now.
host.service_restart('hadoop-yarn-resourcemanager')
host.service_restart('hadoop-mapreduce-historyserver')
for port in get_layer_opts().exposed_ports('resourcemanager'):
hookenv.open_port(port)
set_state('apache-bigtop-resourcemanager.started')
hookenv.status_set('active', 'ready')
def start_resourcemanager(namenode):
hookenv.status_set('maintenance', 'starting resourcemanager')
# NB: service should be started by install, but this may be handy in case
# we have something that removes the .started state in the future. Also
# note we restart here in case we modify conf between install and now.
host.service_restart('hadoop-yarn-resourcemanager')
host.service_restart('hadoop-mapreduce-historyserver')
for port in get_layer_opts().exposed_ports('resourcemanager'):
hookenv.open_port(port)
set_state('apache-bigtop-resourcemanager.started')
hookenv.status_set('active', 'ready')
def start_namenode():
hookenv.status_set('maintenance', 'starting namenode')
# NB: service should be started by install, but we want to verify it is
# running before we set the .started state and open ports. We always
# restart here, which may seem heavy-handed. However, restart works
# whether the service is currently started or stopped. It also ensures the
# service is using the most current config.
started = host.service_restart('hadoop-hdfs-namenode')
if started:
for port in get_layer_opts().exposed_ports('namenode'):
hookenv.open_port(port)
set_state('apache-bigtop-namenode.started')
hookenv.status_set('maintenance', 'namenode started')
hookenv.application_version_set(get_hadoop_version())
else:
hookenv.log('NameNode failed to start')
hookenv.status_set('blocked', 'namenode failed to start')
remove_state('apache-bigtop-namenode.started')
for port in get_layer_opts().exposed_ports('namenode'):
hookenv.close_port(port)
def start_nodemanager(namenode, resourcemanager):
hookenv.status_set('maintenance', 'starting nodemanager')
# NB: service should be started by install, but we want to verify it is
# running before we set the .started state and open ports. We always
# restart here, which may seem heavy-handed. However, restart works
# whether the service is currently started or stopped. It also ensures the
# service is using the most current config.
started = host.service_restart('hadoop-yarn-nodemanager')
if started:
for port in get_layer_opts().exposed_ports('nodemanager'):
hookenv.open_port(port)
set_state('apache-bigtop-nodemanager.started')
hookenv.status_set('maintenance', 'nodemanager started')
hookenv.application_version_set(get_hadoop_version())
else:
hookenv.log('NodeManager failed to start')
hookenv.status_set('blocked', 'nodemanager failed to start')
remove_state('apache-bigtop-nodemanager.started')
for port in get_layer_opts().exposed_ports('nodemanager'):
hookenv.close_port(port)
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
bigtop.render_site_yaml(
hosts={
'namenode': get_fqdn(),
},
roles=[
'namenode',
'mapred-app',
],
# NB: We want the NN to listen on all interfaces, so bind to 0.0.0.0.
overrides={
'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port,
'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port,
'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0',
}
)
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# We need to create the 'mapred' and 'spark' user/group since we may not
# be installing hadoop-mapreduce or spark on this machine. This is needed
# so the namenode can access yarn and spark job history files in hdfs. Also
# add our ubuntu user to the hadoop, mapred, and spark groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
bigtop.render_site_yaml(
hosts={
'namenode': get_fqdn(),
},
roles=[
'namenode',
'mapred-app',
],
# NB: We want the NN to listen on all interfaces, so bind to 0.0.0.0.
overrides={
'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port,
'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port,
'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0',
})
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# We need to create the 'mapred' user/group since we are not installing
# hadoop-mapreduce. This is needed so the namenode can access yarn
# job history files in hdfs. Also add our ubuntu user to the hadoop
# and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def install_namenode():
hookenv.status_set('maintenance', 'installing namenode')
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': get_fqdn(),
},
roles=[
'namenode',
'mapred-app',
],
)
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: dfs-slave)
# to signify NN's readiness. Set our NN info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# make our namenode listen on all interfaces
hdfs_site = Path('/etc/hadoop/conf/hdfs-site.xml')
with utils.xmlpropmap_edit_in_place(hdfs_site) as props:
props['dfs.namenode.rpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.servicerpc-bind-host'] = '0.0.0.0'
props['dfs.namenode.http-bind-host'] = '0.0.0.0'
props['dfs.namenode.https-bind-host'] = '0.0.0.0'
# We need to create the 'mapred' user/group since we are not installing
# hadoop-mapreduce. This is needed so the namenode can access yarn
# job history files in hdfs. Also add our ubuntu user to the hadoop
# and mapred groups.
get_layer_opts().add_users()
set_state('apache-bigtop-namenode.installed')
hookenv.status_set('maintenance', 'namenode installed')
def start_resourcemanager(namenode):
hookenv.status_set('maintenance', 'starting resourcemanager')
# NB: service should be started by install, but we want to verify it is
# running before we set the .started state and open ports. We always
# restart here, which may seem heavy-handed. However, restart works
# whether the service is currently started or stopped. It also ensures the
# service is using the most current config.
rm_started = host.service_restart('hadoop-yarn-resourcemanager')
if rm_started:
for port in get_layer_opts().exposed_ports('resourcemanager'):
hookenv.open_port(port)
set_state('apache-bigtop-resourcemanager.started')
hookenv.status_set('maintenance', 'resourcemanager started')
hookenv.application_version_set(get_hadoop_version())
else:
hookenv.log('YARN ResourceManager failed to start')
hookenv.status_set('blocked', 'resourcemanager failed to start')
remove_state('apache-bigtop-resourcemanager.started')
for port in get_layer_opts().exposed_ports('resourcemanager'):
hookenv.close_port(port)
hs_started = host.service_restart('hadoop-mapreduce-historyserver')
if not hs_started:
hookenv.log('YARN HistoryServer failed to start')
def stop_nodemanager():
hookenv.status_set('maintenance', 'stopping nodemanager')
stopped = host.service_stop('hadoop-yarn-nodemanager')
if stopped:
hookenv.status_set('maintenance', 'nodemanager stopped')
else:
hookenv.log('NodeManager failed to stop')
hookenv.status_set('blocked', 'nodemanager failed to stop')
# Even if the service failed to stop, we want to treat it as stopped so
# other apps do not attempt to interact with it. Remove .started and
# close our ports.
remove_state('apache-bigtop-nodemanager.started')
for port in get_layer_opts().exposed_ports('nodemanager'):
hookenv.close_port(port)
def install_component(namenode, resourcemanager):
'''Install only if nn and rm have sent their FQDNs.'''
if namenode.namenodes() and resourcemanager.resourcemanagers():
set_state('components.installed')
options = layer.options('apache-bigtop-base')
components = options.get("bigtop_component_list")
hookenv.status_set('maintenance', 'installing {}'.format(components))
nn_fqdn = namenode.namenodes()[0]
rm_fqdn = resourcemanager.resourcemanagers()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_fqdn, 'resourcemanager': rm_fqdn}
bigtop.install_component(RM=rm_fqdn, NN=nn_fqdn)
for component in components.split():
try:
for port in get_layer_opts().exposed_ports(component):
hookenv.open_port(port)
except AttributeError:
hookenv.log("Not opening ports for component {}".format(component))
hookenv.status_set('active', 'ready')
else:
hookenv.status_set('waiting',
'waiting for namenode and resource manager to become ready')
def install_component(namenode, resourcemanager):
'''Install only if nn and rm have sent their FQDNs.'''
if namenode.namenodes() and resourcemanager.resourcemanagers():
set_state('components.installed')
options = layer.options('apache-bigtop-base')
components = options.get("bigtop_component_list")
hookenv.status_set('maintenance', 'installing {}'.format(components))
nn_fqdn = namenode.namenodes()[0]
rm_fqdn = resourcemanager.resourcemanagers()[0]
bigtop = Bigtop()
hosts = {'namenode': nn_fqdn, 'resourcemanager': rm_fqdn}
bigtop.install_component(RM=rm_fqdn, NN=nn_fqdn)
for component in components.split():
try:
for port in get_layer_opts().exposed_ports(component):
hookenv.open_port(port)
except AttributeError:
hookenv.log(
"Not opening ports for component {}".format(component))
hookenv.status_set('active', 'ready')
else:
hookenv.status_set(
'waiting',
'waiting for namenode and resource manager to become ready')
def install_resourcemanager(namenode):
"""Install if the namenode has sent its FQDN.
We only need the namenode FQDN to perform the RM install, so poll for
namenodes() data whenever we have a namenode relation. This allows us to
install asap, even if 'namenode.ready' is not set yet.
"""
if namenode.namenodes():
hookenv.status_set('maintenance', 'installing resourcemanager')
# Hosts
nn_host = namenode.namenodes()[0]
rm_host = get_fqdn()
# Ports
rm_ipc = get_layer_opts().port('resourcemanager')
rm_http = get_layer_opts().port('rm_webapp_http')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
hdfs_port = namenode.port()
webhdfs_port = namenode.webhdfs_port()
bigtop = Bigtop()
bigtop.render_site_yaml(
hosts={
'namenode': nn_host,
'resourcemanager': rm_host,
},
roles=[
'resourcemanager',
],
# NB: When we colocate the NN and RM, the RM will run puppet apply
# last. To ensure we don't lose any hdfs-site.xml data set by the
# NN, override common_hdfs properties again here.
overrides={
'hadoop::common_yarn::hadoop_rm_port': rm_ipc,
'hadoop::common_yarn::hadoop_rm_webapp_port': rm_http,
'hadoop::common_yarn::hadoop_rm_bind_host': '0.0.0.0',
'hadoop::common_mapred_app::mapreduce_jobhistory_host': '0.0.0.0',
'hadoop::common_mapred_app::mapreduce_jobhistory_port': jh_ipc,
'hadoop::common_mapred_app::mapreduce_jobhistory_webapp_port': jh_http,
'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port,
'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port,
'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0',
'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0',
}
)
bigtop.trigger_puppet()
# /etc/hosts entries from the KV are not currently used for bigtop,
# but a hosts_map attribute is required by some interfaces (eg: mapred-slave)
# to signify RM's readiness. Set our RM info in the KV to fulfill this
# requirement.
utils.initialize_kv_host()
# We need to create the 'spark' user/group since we may not be
# installing spark on this machine. This is needed so the history
# server can access spark job history files in hdfs. Also add our
# ubuntu user to the hadoop, mapred, and spark groups on this machine.
get_layer_opts().add_users()
set_state('apache-bigtop-resourcemanager.installed')
hookenv.status_set('maintenance', 'resourcemanager installed')
else:
hookenv.status_set('waiting', 'waiting for namenode fqdn')
def stop_nodemanager():
hookenv.status_set('maintenance', 'stopping nodemanager')
for port in get_layer_opts().exposed_ports('nodemanager'):
host.service_stop('hadoop-yarn-nodemanager')
remove_state('nodemanager.started')
hookenv.status_set('maintenance', 'nodemanager stopped')
def test_get_layer_opts(self, mock_options):
'''Verify that we parse whatever dict we get back from options.'''
mock_options.get.return_value = {'foo': 'bar'}
ret = get_layer_opts()
self.assertEqual(ret.dist_config['foo'], 'bar')
def test_get_layer_opts(self, mock_options):
'''Verify that we parse whatever dict we get back from options.'''
mock_options.return_value = {'foo': 'bar'}
ret = get_layer_opts()
self.assertEqual(ret.dist_config['foo'], 'bar')
def serve_client(client, zookeeper):
kafka_port = get_layer_opts().port('kafka')
client.send_port(kafka_port)
client.send_zookeepers(zookeeper.zookeepers())
hookenv.log('Sent Kafka configuration to client')
def serve_client(client):
config = get_layer_opts()
master_port = config.port('hbase-master')
regionserver_port = config.port('hbase-region')
thrift_port = config.port('hbase-thrift')
client.send_port(master_port, regionserver_port, thrift_port)
def serve_client(client):
config = get_layer_opts()
master_port = config.port('hbase-master')
regionserver_port = config.port('hbase-region')
thrift_port = config.port('hbase-thrift')
client.send_port(master_port, regionserver_port, thrift_port)
def serve_client(client, zookeeper):
kafka_port = get_layer_opts().port('kafka')
client.send_port(kafka_port)
client.send_zookeepers(zookeeper.zookeepers())
hookenv.log('Sent Kafka configuration to client')