def send_info(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') port = hadoop.dist_config.port('resourcemanager') hs_http = hadoop.dist_config.port('jh_webapp_http') hs_ipc = hadoop.dist_config.port('jobhistory') utils.update_kv_hosts(nodemanager.hosts_map()) utils.manage_etc_hosts() nodemanager.send_spec(hadoop.spec()) nodemanager.send_resourcemanagers([local_hostname]) nodemanager.send_ports(port, hs_http, hs_ipc) nodemanager.send_ssh_key(utils.get_ssh_key('yarn')) nodemanager.send_hosts_map(utils.get_kv_hosts()) slaves = nodemanager.nodes() if data_changed('resourcemanager.slaves', slaves): unitdata.kv().set('resourcemanager.slaves', slaves) yarn.register_slaves(slaves) hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('resourcemanager.ready')
def send_info(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') port = hadoop.dist_config.port('resourcemanager') hs_http = hadoop.dist_config.port('jh_webapp_http') hs_ipc = hadoop.dist_config.port('jobhistory') utils.update_kv_hosts({node['ip']: node['host'] for node in nodemanager.nodes()}) utils.manage_etc_hosts() nodemanager.send_spec(hadoop.spec()) nodemanager.send_resourcemanagers([local_hostname]) nodemanager.send_ports(port, hs_http, hs_ipc) nodemanager.send_ssh_key(utils.get_ssh_key('hdfs')) nodemanager.send_hosts_map(utils.get_kv_hosts()) slaves = [node['host'] for node in nodemanager.nodes()] if data_changed('resourcemanager.slaves', slaves): unitdata.kv().set('resourcemanager.slaves', slaves) yarn.register_slaves(slaves) hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('resourcemanager.ready')
def unregister_nodemanager(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) slaves = unitdata.kv().get('resourcemanager.slaves', []) slaves_leaving = nodemanager.nodes() hookenv.log('Slaves leaving: {}'.format(slaves_leaving)) slaves_remaining = list(set(slaves) - set(slaves_leaving)) unitdata.kv().set('resourcemanager.slaves', slaves_remaining) yarn.register_slaves(slaves_remaining) utils.remove_kv_hosts(slaves_leaving) utils.manage_etc_hosts() if not slaves_remaining: remove_state('resourcemanager.ready') nodemanager.dismiss()
def unregister_nodemanager(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) nodes_leaving = nodemanager.nodes() # only returns nodes in "leaving" state slaves = unitdata.kv().get('resourcemanager.slaves', []) slaves_leaving = [node['host'] for node in nodes_leaving] hookenv.log('Slaves leaving: {}'.format(slaves_leaving)) slaves_remaining = list(set(slaves) - set(slaves_leaving)) unitdata.kv().set('resourcemanager.slaves', slaves_remaining) yarn.register_slaves(slaves_remaining) utils.remove_kv_hosts(slaves_leaving) utils.manage_etc_hosts() if not slaves_remaining: remove_state('resourcemanager.ready') nodemanager.dismiss()