def configure_yarn(resourcemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) utils.update_kv_hosts(resourcemanager.hosts_map()) utils.manage_etc_hosts() if not resourcemanager.resourcemanagers(): data = yaml.dump( { 'relation_name': resourcemanager.relation_name, 'conversations': { conv.key: dict({'relation_ids': conv.relation_ids}, **conv.serialize(conv)) for conv in resourcemanager.conversations() }, 'relation_data': { rid: { unit: hookenv.relation_get(unit=unit, rid=rid) for unit in hookenv.related_units(rid) } for rid in hookenv.relation_ids( resourcemanager.relation_name) }, }, default_flow_style=False) for line in data.splitlines(): hookenv.log(line) yarn.configure_yarn_base(resourcemanager.resourcemanagers()[0], resourcemanager.port(), resourcemanager.hs_http(), resourcemanager.hs_ipc()) set_state('hadoop.yarn.configured')
def send_info(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') port = hadoop.dist_config.port('resourcemanager') hs_http = hadoop.dist_config.port('jh_webapp_http') hs_ipc = hadoop.dist_config.port('jobhistory') utils.update_kv_hosts({node['ip']: node['host'] for node in nodemanager.nodes()}) utils.manage_etc_hosts() nodemanager.send_spec(hadoop.spec()) nodemanager.send_resourcemanagers([local_hostname]) nodemanager.send_ports(port, hs_http, hs_ipc) nodemanager.send_ssh_key(utils.get_ssh_key('hdfs')) nodemanager.send_hosts_map(utils.get_kv_hosts()) slaves = [node['host'] for node in nodemanager.nodes()] if data_changed('resourcemanager.slaves', slaves): unitdata.kv().set('resourcemanager.slaves', slaves) yarn.register_slaves(slaves) hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('resourcemanager.ready')
def configure_hdfs(hdfs_rel): hadoop = get_hadoop_base() yarn = YARN(hadoop) yarn.start_resourcemanager() yarn.start_jobhistory() hadoop.open_ports('resourcemanager') set_state('resourcemanager.started')
def hdfs_departed(): hadoop = get_hadoop_base() yarn = YARN(hadoop) hadoop.close_ports('resourcemanager') yarn.stop_jobhistory() yarn.stop_resourcemanager() remove_state('resourcemanager.started') remove_state('resourcemanager.ready')
def configure_resourcemanager(): local_hostname = hookenv.local_unit().replace('/', '-') private_address = hookenv.unit_get('private-address') ip_addr = utils.resolve_private_address(private_address) hadoop = get_hadoop_base() yarn = YARN(hadoop) yarn.configure_resourcemanager() yarn.configure_jobhistory() utils.update_kv_hosts({ip_addr: local_hostname}) set_state('resourcemanager.configured')
def start_nodemanager(resourcemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) yarn.configure_nodemanager(resourcemanager.resourcemanagers()[0], resourcemanager.port(), resourcemanager.hs_http(), resourcemanager.hs_ipc()) utils.install_ssh_key('yarn', resourcemanager.ssh_key()) utils.update_kv_hosts(resourcemanager.hosts_map()) utils.manage_etc_hosts() yarn.start_nodemanager() hadoop.open_ports('nodemanager') set_state('nodemanager.started')
def ganglia_changed(): hadoop = get_hadoop_base() hdfs = HDFS(hadoop) yarn = YARN(hadoop) if is_state('namenode.started'): hdfs.restart_namenode() if is_state('datanode.started'): hdfs.restart_datanode() if is_state('journalnode.started'): hdfs.restart_journalnode() if is_state('resourcemanager.started'): yarn.restart_resourcemanager() if is_state('nodemanager.started'): yarn.restart_nodemanager()
def unregister_nodemanager(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) nodes_leaving = nodemanager.nodes() # only returns nodes in "leaving" state slaves = unitdata.kv().get('resourcemanager.slaves', []) slaves_leaving = [node['host'] for node in nodes_leaving] hookenv.log('Slaves leaving: {}'.format(slaves_leaving)) slaves_remaining = list(set(slaves) - set(slaves_leaving)) unitdata.kv().set('resourcemanager.slaves', slaves_remaining) yarn.register_slaves(slaves_remaining) utils.remove_kv_hosts(slaves_leaving) utils.manage_etc_hosts() if not slaves_remaining: remove_state('resourcemanager.ready') nodemanager.dismiss()
def stop_nodemanager(): hadoop = get_hadoop_base() yarn = YARN(hadoop) yarn.stop_nodemanager() hadoop.close_ports('nodemanager') remove_state('nodemanager.started')