def send_nm_all_info(nodemanager):
"""Send nodemanagers all mapred-slave relation data.
At this point, the resourcemanager is ready to serve nodemanagers. Send all
mapred-slave relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
nodemanager.send_resourcemanagers([rm_host])
nodemanager.send_spec(bigtop.spec())
nodemanager.send_ports(rm_ipc, jh_http, jh_ipc)
# hosts_map and ssh_key are required by the mapred-slave interface to signify
# RM's readiness. Send them, even though they are not utilized by bigtop.
# NB: update KV hosts with all nodemanagers prior to sending the hosts_map
# because mapred-slave gates readiness on a NM's presence in the hosts_map.
utils.update_kv_hosts(nodemanager.hosts_map())
nodemanager.send_hosts_map(utils.get_kv_hosts())
nodemanager.send_ssh_key('invalid')
# update status with slave count and report ready for hdfs
num_slaves = len(nodemanager.nodes())
hookenv.status_set('active', 'ready ({count} nodemanager{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-resourcemanager.ready')
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the resourcemanager is ready to serve clients. Send all
mapred relation data so that our 'resourcemanager.ready' state becomes set.
"""
bigtop = Bigtop()
rm_host = get_fqdn()
rm_ipc = get_layer_opts().port('resourcemanager')
jh_ipc = get_layer_opts().port('jobhistory')
jh_http = get_layer_opts().port('jh_webapp_http')
client.send_resourcemanagers([rm_host])
client.send_spec(bigtop.spec())
client.send_ports(rm_ipc, jh_http, jh_ipc)
# resourcemanager.ready implies we have at least 1 nodemanager, which means
# yarn is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-resourcemanager.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map is required by the mapred interface to signify
# RM's readiness. Send it, even though it is not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
def send_dn_all_info(datanode):
"""Send datanodes all dfs-slave relation data.
At this point, the namenode is ready to serve datanodes. Send all
dfs-slave relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
datanode.send_spec(bigtop.spec())
datanode.send_namenodes([fqdn])
datanode.send_ports(hdfs_port, webhdfs_port)
# hosts_map, ssh_key, and clustername are required by the dfs-slave
# interface to signify NN's readiness. Send them, even though they are not
# utilized by bigtop.
# NB: update KV hosts with all datanodes prior to sending the hosts_map
# because dfs-slave gates readiness on a DN's presence in the hosts_map.
utils.update_kv_hosts(datanode.hosts_map())
datanode.send_hosts_map(utils.get_kv_hosts())
datanode.send_ssh_key('invalid')
datanode.send_clustername(hookenv.service_name())
# update status with slave count and report ready for hdfs
num_slaves = len(datanode.nodes())
hookenv.status_set(
'active', 'ready ({count} datanode{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-namenode.ready')
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the namenode is ready to serve clients. Send all
dfs relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
client.send_spec(bigtop.spec())
client.send_namenodes([fqdn])
client.send_ports(hdfs_port, webhdfs_port)
# namenode.ready implies we have at least 1 datanode, which means hdfs
# is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-namenode.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map and clustername are required by the dfs interface to signify
# NN's readiness. Send it, even though they are not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
client.send_clustername(hookenv.service_name())
def send_client_all_info(client):
"""Send clients (plugin, RM, non-DNs) all dfs relation data.
At this point, the namenode is ready to serve clients. Send all
dfs relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
client.send_spec(bigtop.spec())
client.send_namenodes([fqdn])
client.send_ports(hdfs_port, webhdfs_port)
# namenode.ready implies we have at least 1 datanode, which means hdfs
# is ready for use. Inform clients of that with send_ready().
if is_state('apache-bigtop-namenode.ready'):
client.send_ready(True)
else:
client.send_ready(False)
# hosts_map and clustername are required by the dfs interface to signify
# NN's readiness. Send it, even though they are not utilized by bigtop.
client.send_hosts_map(utils.get_kv_hosts())
client.send_clustername(hookenv.service_name())
def send_dn_all_info(datanode):
"""Send datanodes all dfs-slave relation data.
At this point, the namenode is ready to serve datanodes. Send all
dfs-slave relation data so that our 'namenode.ready' state becomes set.
"""
bigtop = Bigtop()
fqdn = get_fqdn()
hdfs_port = get_layer_opts().port('namenode')
webhdfs_port = get_layer_opts().port('nn_webapp_http')
datanode.send_spec(bigtop.spec())
datanode.send_namenodes([fqdn])
datanode.send_ports(hdfs_port, webhdfs_port)
# hosts_map, ssh_key, and clustername are required by the dfs-slave
# interface to signify NN's readiness. Send them, even though they are not
# utilized by bigtop.
# NB: update KV hosts with all datanodes prior to sending the hosts_map
# because dfs-slave gates readiness on a DN's presence in the hosts_map.
utils.update_kv_hosts(datanode.hosts_map())
datanode.send_hosts_map(utils.get_kv_hosts())
datanode.send_ssh_key('invalid')
datanode.send_clustername(hookenv.service_name())
# update status with slave count and report ready for hdfs
num_slaves = len(datanode.nodes())
hookenv.status_set('active', 'ready ({count} datanode{s})'.format(
count=num_slaves,
s='s' if num_slaves > 1 else '',
))
set_state('apache-bigtop-namenode.ready')
def send_nn_spec(namenode):
"""Send our resourcemanager spec so the namenode can become ready."""
bigtop = Bigtop()
namenode.set_local_spec(bigtop.spec())
hookenv.status_set('waiting', 'waiting for namenode to become ready')
def send_nm_spec(namenode, resourcemanager):
"""Send our nodemanager spec so the master relations can become ready."""
bigtop = Bigtop()
namenode.set_local_spec(bigtop.spec())
resourcemanager.set_local_spec(bigtop.spec())
def send_rm_spec(principal, resourcemanager):
"""Send our plugin spec so the resourcemanager can become ready."""
bigtop = Bigtop()
resourcemanager.set_local_spec(bigtop.spec())
def send_nn_spec(principal, namenode):
"""Send our plugin spec so the namenode can become ready."""
bigtop = Bigtop()
# Send plugin spec (must match NN spec for 'namenode.ready' to be set)
namenode.set_local_spec(bigtop.spec())
def send_rm_spec(principal, resourcemanager):
"""Send our plugin spec so the resourcemanager can become ready."""
bigtop = Bigtop()
resourcemanager.set_local_spec(bigtop.spec())
def send_nn_spec(principal, namenode):
"""Send our plugin spec so the namenode can become ready."""
bigtop = Bigtop()
# Send plugin spec (must match NN spec for 'namenode.ready' to be set)
namenode.set_local_spec(bigtop.spec())
def send_nn_spec(namenode):
"""Send our datanode spec so the namenode can become ready."""
bigtop = Bigtop()
namenode.set_local_spec(bigtop.spec())
