def send_info(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
local_hostname = hookenv.local_unit().replace("/", "-")
hdfs_port = hadoop.dist_config.port("namenode")
webhdfs_port = hadoop.dist_config.port("nn_webapp_http")
utils.update_kv_hosts({node["ip"]: node["host"] for node in datanode.nodes()})
utils.manage_etc_hosts()
datanode.send_spec(hadoop.spec())
datanode.send_namenodes([local_hostname])
datanode.send_ports(hdfs_port, webhdfs_port)
datanode.send_ssh_key(utils.get_ssh_key("hdfs"))
datanode.send_hosts_map(utils.get_kv_hosts())
slaves = [node["host"] for node in datanode.nodes()]
if data_changed("namenode.slaves", slaves):
unitdata.kv().set("namenode.slaves", slaves)
hdfs.register_slaves(slaves)
hookenv.status_set(
"active", "Ready ({count} DataNode{s})".format(count=len(slaves), s="s" if len(slaves) > 1 else "")
)
set_state("namenode.ready")
def send_info(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
local_hostname = hookenv.local_unit().replace('/', '-')
hdfs_port = hadoop.dist_config.port('namenode')
webhdfs_port = hadoop.dist_config.port('nn_webapp_http')
utils.update_kv_hosts(datanode.hosts_map())
utils.manage_etc_hosts()
datanode.send_spec(hadoop.spec())
datanode.send_namenodes([local_hostname])
datanode.send_ports(hdfs_port, webhdfs_port)
datanode.send_ssh_key(utils.get_ssh_key('hdfs'))
datanode.send_hosts_map(utils.get_kv_hosts())
slaves = datanode.nodes()
if data_changed('namenode.slaves', slaves):
unitdata.kv().set('namenode.slaves', slaves)
hdfs.register_slaves(slaves)
hdfs.refresh_slaves()
hookenv.status_set('active', 'Ready ({count} DataNode{s})'.format(
count=len(slaves),
s='s' if len(slaves) > 1 else '',
))
set_state('namenode.ready')
def send_info(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
local_hostname = hookenv.local_unit().replace('/', '-')
hdfs_port = hadoop.dist_config.port('namenode')
webhdfs_port = hadoop.dist_config.port('nn_webapp_http')
utils.update_kv_hosts(
{node['ip']: node['host']
for node in datanode.nodes()})
utils.manage_etc_hosts()
datanode.send_spec(hadoop.spec())
datanode.send_namenodes([local_hostname])
datanode.send_ports(hdfs_port, webhdfs_port)
datanode.send_ssh_key(utils.get_ssh_key('hdfs'))
datanode.send_hosts_map(utils.get_kv_hosts())
slaves = [node['host'] for node in datanode.nodes()]
if data_changed('namenode.slaves', slaves):
unitdata.kv().set('namenode.slaves', slaves)
hdfs.register_slaves(slaves)
hookenv.status_set(
'active', 'Ready ({count} DataNode{s})'.format(
count=len(slaves),
s='s' if len(slaves) > 1 else '',
))
set_state('namenode.ready')
def update_slaves(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
slaves = datanode.nodes()
if data_changed('namenode.slaves', slaves):
unitdata.kv().set('namenode.slaves', slaves)
hdfs.register_slaves(slaves)
hdfs.reload_slaves()
set_state('namenode.ready')
def unregister_datanode(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
slaves = unitdata.kv().get('namenode.slaves', [])
slaves_leaving = datanode.nodes() # only returns nodes in "leaving" state
hookenv.log('Slaves leaving: {}'.format(slaves_leaving))
slaves_remaining = list(set(slaves) - set(slaves_leaving))
unitdata.kv().set('namenode.slaves', slaves_remaining)
hdfs.register_slaves(slaves_remaining)
hdfs.reload_slaves()
utils.remove_kv_hosts(slaves_leaving)
utils.manage_etc_hosts()
if not slaves_remaining:
remove_state('namenode.ready')
datanode.dismiss()
def unregister_datanode(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
nodes_leaving = datanode.nodes() # only returns nodes in "leaving" state
slaves = unitdata.kv().get("namenode.slaves", [])
slaves_leaving = [node["host"] for node in nodes_leaving]
hookenv.log("Slaves leaving: {}".format(slaves_leaving))
slaves_remaining = list(set(slaves) - set(slaves_leaving))
unitdata.kv().set("namenode.slaves", slaves_remaining)
hdfs.register_slaves(slaves_remaining)
utils.remove_kv_hosts(slaves_leaving)
utils.manage_etc_hosts()
if not slaves_remaining:
hookenv.status_set("blocked", "Waiting for relation to DataNodes")
remove_state("namenode.ready")
datanode.dismiss()
def unregister_datanode(datanode):
hadoop = get_hadoop_base()
hdfs = HDFS(hadoop)
nodes_leaving = datanode.nodes() # only returns nodes in "leaving" state
slaves = unitdata.kv().get('namenode.slaves', [])
slaves_leaving = [node['host'] for node in nodes_leaving]
hookenv.log('Slaves leaving: {}'.format(slaves_leaving))
slaves_remaining = list(set(slaves) - set(slaves_leaving))
unitdata.kv().set('namenode.slaves', slaves_remaining)
hdfs.register_slaves(slaves_remaining)
utils.remove_kv_hosts(slaves_leaving)
utils.manage_etc_hosts()
if not slaves_remaining:
hookenv.status_set('blocked', 'Waiting for relation to DataNodes')
remove_state('namenode.ready')
datanode.dismiss()
