def send_info(nodemanager):
hadoop = get_hadoop_base()
yarn = YARN(hadoop)
local_hostname = hookenv.local_unit().replace('/', '-')
port = hadoop.dist_config.port('resourcemanager')
hs_http = hadoop.dist_config.port('jh_webapp_http')
hs_ipc = hadoop.dist_config.port('jobhistory')
utils.update_kv_hosts(nodemanager.hosts_map())
utils.manage_etc_hosts()
nodemanager.send_spec(hadoop.spec())
nodemanager.send_resourcemanagers([local_hostname])
nodemanager.send_ports(port, hs_http, hs_ipc)
nodemanager.send_ssh_key(utils.get_ssh_key('yarn'))
nodemanager.send_hosts_map(utils.get_kv_hosts())
slaves = nodemanager.nodes()
if data_changed('resourcemanager.slaves', slaves):
unitdata.kv().set('resourcemanager.slaves', slaves)
yarn.register_slaves(slaves)
hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format(
count=len(slaves),
s='s' if len(slaves) > 1 else '',
))
set_state('resourcemanager.ready')
def send_info(nodemanager):
hadoop = get_hadoop_base()
yarn = YARN(hadoop)
local_hostname = hookenv.local_unit().replace('/', '-')
port = hadoop.dist_config.port('resourcemanager')
hs_http = hadoop.dist_config.port('jh_webapp_http')
hs_ipc = hadoop.dist_config.port('jobhistory')
utils.update_kv_hosts({node['ip']: node['host'] for node in nodemanager.nodes()})
utils.manage_etc_hosts()
nodemanager.send_spec(hadoop.spec())
nodemanager.send_resourcemanagers([local_hostname])
nodemanager.send_ports(port, hs_http, hs_ipc)
nodemanager.send_ssh_key(utils.get_ssh_key('hdfs'))
nodemanager.send_hosts_map(utils.get_kv_hosts())
slaves = [node['host'] for node in nodemanager.nodes()]
if data_changed('resourcemanager.slaves', slaves):
unitdata.kv().set('resourcemanager.slaves', slaves)
yarn.register_slaves(slaves)
hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format(
count=len(slaves),
s='s' if len(slaves) > 1 else '',
))
set_state('resourcemanager.ready')
def unregister_nodemanager(nodemanager):
hadoop = get_hadoop_base()
yarn = YARN(hadoop)
slaves = unitdata.kv().get('resourcemanager.slaves', [])
slaves_leaving = nodemanager.nodes()
hookenv.log('Slaves leaving: {}'.format(slaves_leaving))
slaves_remaining = list(set(slaves) - set(slaves_leaving))
unitdata.kv().set('resourcemanager.slaves', slaves_remaining)
yarn.register_slaves(slaves_remaining)
utils.remove_kv_hosts(slaves_leaving)
utils.manage_etc_hosts()
if not slaves_remaining:
remove_state('resourcemanager.ready')
nodemanager.dismiss()
def unregister_nodemanager(nodemanager):
hadoop = get_hadoop_base()
yarn = YARN(hadoop)
nodes_leaving = nodemanager.nodes() # only returns nodes in "leaving" state
slaves = unitdata.kv().get('resourcemanager.slaves', [])
slaves_leaving = [node['host'] for node in nodes_leaving]
hookenv.log('Slaves leaving: {}'.format(slaves_leaving))
slaves_remaining = list(set(slaves) - set(slaves_leaving))
unitdata.kv().set('resourcemanager.slaves', slaves_remaining)
yarn.register_slaves(slaves_remaining)
utils.remove_kv_hosts(slaves_leaving)
utils.manage_etc_hosts()
if not slaves_remaining:
remove_state('resourcemanager.ready')
nodemanager.dismiss()
