def send_client_all_info(client): """Send clients (plugin, RM, non-DNs) all dfs relation data. At this point, the resourcemanager is ready to serve clients. Send all mapred relation data so that our 'resourcemanager.ready' state becomes set. """ bigtop = Bigtop() rm_host = get_fqdn() rm_ipc = get_layer_opts().port('resourcemanager') jh_ipc = get_layer_opts().port('jobhistory') jh_http = get_layer_opts().port('jh_webapp_http') client.send_resourcemanagers([rm_host]) client.send_spec(bigtop.spec()) client.send_ports(rm_ipc, jh_http, jh_ipc) # resourcemanager.ready implies we have at least 1 nodemanager, which means # yarn is ready for use. Inform clients of that with send_ready(). if is_state('apache-bigtop-resourcemanager.ready'): client.send_ready(True) else: client.send_ready(False) # hosts_map is required by the mapred interface to signify # RM's readiness. Send it, even though it is not utilized by bigtop. client.send_hosts_map(utils.get_kv_hosts())
def send_info(datanode): hadoop = get_hadoop_base() hdfs = HDFS(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') hdfs_port = hadoop.dist_config.port('namenode') webhdfs_port = hadoop.dist_config.port('nn_webapp_http') utils.update_kv_hosts( {node['ip']: node['host'] for node in datanode.nodes()}) utils.manage_etc_hosts() datanode.send_spec(hadoop.spec()) datanode.send_namenodes([local_hostname]) datanode.send_ports(hdfs_port, webhdfs_port) datanode.send_ssh_key(utils.get_ssh_key('hdfs')) datanode.send_hosts_map(utils.get_kv_hosts()) slaves = [node['host'] for node in datanode.nodes()] if data_changed('namenode.slaves', slaves): unitdata.kv().set('namenode.slaves', slaves) hdfs.register_slaves(slaves) hookenv.status_set( 'active', 'Ready ({count} DataNode{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('namenode.ready')
def send_client_all_info(client): """Send clients (plugin, RM, non-DNs) all dfs relation data. At this point, the namenode is ready to serve clients. Send all dfs relation data so that our 'namenode.ready' state becomes set. """ bigtop = Bigtop() fqdn = get_fqdn() hdfs_port = get_layer_opts().port('namenode') webhdfs_port = get_layer_opts().port('nn_webapp_http') client.send_spec(bigtop.spec()) client.send_namenodes([fqdn]) client.send_ports(hdfs_port, webhdfs_port) # namenode.ready implies we have at least 1 datanode, which means hdfs # is ready for use. Inform clients of that with send_ready(). if is_state('apache-bigtop-namenode.ready'): client.send_ready(True) else: client.send_ready(False) # hosts_map and clustername are required by the dfs interface to signify # NN's readiness. Send it, even though they are not utilized by bigtop. client.send_hosts_map(utils.get_kv_hosts()) client.send_clustername(hookenv.service_name())
def send_dn_all_info(datanode): """Send datanodes all dfs-slave relation data. At this point, the namenode is ready to serve datanodes. Send all dfs-slave relation data so that our 'namenode.ready' state becomes set. """ bigtop = Bigtop() fqdn = get_fqdn() hdfs_port = get_layer_opts().port('namenode') webhdfs_port = get_layer_opts().port('nn_webapp_http') datanode.send_spec(bigtop.spec()) datanode.send_namenodes([fqdn]) datanode.send_ports(hdfs_port, webhdfs_port) # hosts_map, ssh_key, and clustername are required by the dfs-slave # interface to signify NN's readiness. Send them, even though they are not # utilized by bigtop. # NB: update KV hosts with all datanodes prior to sending the hosts_map # because dfs-slave gates readiness on a DN's presence in the hosts_map. utils.update_kv_hosts(datanode.hosts_map()) datanode.send_hosts_map(utils.get_kv_hosts()) datanode.send_ssh_key('invalid') datanode.send_clustername(hookenv.service_name()) # update status with slave count and report ready for hdfs num_slaves = len(datanode.nodes()) hookenv.status_set( 'active', 'ready ({count} datanode{s})'.format( count=num_slaves, s='s' if num_slaves > 1 else '', )) set_state('apache-bigtop-namenode.ready')
def send_info(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') port = hadoop.dist_config.port('resourcemanager') hs_http = hadoop.dist_config.port('jh_webapp_http') hs_ipc = hadoop.dist_config.port('jobhistory') utils.update_kv_hosts({node['ip']: node['host'] for node in nodemanager.nodes()}) utils.manage_etc_hosts() nodemanager.send_spec(hadoop.spec()) nodemanager.send_resourcemanagers([local_hostname]) nodemanager.send_ports(port, hs_http, hs_ipc) nodemanager.send_ssh_key(utils.get_ssh_key('hdfs')) nodemanager.send_hosts_map(utils.get_kv_hosts()) slaves = [node['host'] for node in nodemanager.nodes()] if data_changed('resourcemanager.slaves', slaves): unitdata.kv().set('resourcemanager.slaves', slaves) yarn.register_slaves(slaves) hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('resourcemanager.ready')
def send_dn_all_info(datanode): """Send datanodes all dfs-slave relation data. At this point, the namenode is ready to serve datanodes. Send all dfs-slave relation data so that our 'namenode.ready' state becomes set. """ bigtop = Bigtop() fqdn = get_fqdn() hdfs_port = get_layer_opts().port('namenode') webhdfs_port = get_layer_opts().port('nn_webapp_http') datanode.send_spec(bigtop.spec()) datanode.send_namenodes([fqdn]) datanode.send_ports(hdfs_port, webhdfs_port) # hosts_map, ssh_key, and clustername are required by the dfs-slave # interface to signify NN's readiness. Send them, even though they are not # utilized by bigtop. # NB: update KV hosts with all datanodes prior to sending the hosts_map # because dfs-slave gates readiness on a DN's presence in the hosts_map. utils.update_kv_hosts(datanode.hosts_map()) datanode.send_hosts_map(utils.get_kv_hosts()) datanode.send_ssh_key('invalid') datanode.send_clustername(hookenv.service_name()) # update status with slave count and report ready for hdfs num_slaves = len(datanode.nodes()) hookenv.status_set('active', 'ready ({count} datanode{s})'.format( count=num_slaves, s='s' if num_slaves > 1 else '', )) set_state('apache-bigtop-namenode.ready')
def send_info(datanode): hadoop = get_hadoop_base() hdfs = HDFS(hadoop) local_hostname = hookenv.local_unit().replace("/", "-") hdfs_port = hadoop.dist_config.port("namenode") webhdfs_port = hadoop.dist_config.port("nn_webapp_http") utils.update_kv_hosts({node["ip"]: node["host"] for node in datanode.nodes()}) utils.manage_etc_hosts() datanode.send_spec(hadoop.spec()) datanode.send_namenodes([local_hostname]) datanode.send_ports(hdfs_port, webhdfs_port) datanode.send_ssh_key(utils.get_ssh_key("hdfs")) datanode.send_hosts_map(utils.get_kv_hosts()) slaves = [node["host"] for node in datanode.nodes()] if data_changed("namenode.slaves", slaves): unitdata.kv().set("namenode.slaves", slaves) hdfs.register_slaves(slaves) hookenv.status_set( "active", "Ready ({count} DataNode{s})".format(count=len(slaves), s="s" if len(slaves) > 1 else "") ) set_state("namenode.ready")
def send_info(datanode): hadoop = get_hadoop_base() hdfs = HDFS(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') hdfs_port = hadoop.dist_config.port('namenode') webhdfs_port = hadoop.dist_config.port('nn_webapp_http') utils.update_kv_hosts(datanode.hosts_map()) utils.manage_etc_hosts() datanode.send_spec(hadoop.spec()) datanode.send_namenodes([local_hostname]) datanode.send_ports(hdfs_port, webhdfs_port) datanode.send_ssh_key(utils.get_ssh_key('hdfs')) datanode.send_hosts_map(utils.get_kv_hosts()) slaves = datanode.nodes() if data_changed('namenode.slaves', slaves): unitdata.kv().set('namenode.slaves', slaves) hdfs.register_slaves(slaves) hdfs.refresh_slaves() hookenv.status_set('active', 'Ready ({count} DataNode{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('namenode.ready')
def send_info(datanode): hadoop = get_bigtop_base() # hdfs = HDFS(hadoop) # local_hostname = hookenv.local_unit().replace('/', '-') # hdfs_port = hadoop.dist_config.port('namenode') # webhdfs_port = hadoop.dist_config.port('nn_webapp_http') utils.update_kv_hosts({node['ip']: node['host'] for node in datanode.nodes()}) utils.manage_etc_hosts() # datanode.send_spec(hadoop.spec()) # datanode.send_namenodes([local_hostname]) # datanode.send_ports(hdfs_port, webhdfs_port) # datanode.send_ssh_key(utils.get_ssh_key('hdfs')) datanode.send_hosts_map(utils.get_kv_hosts()) # slaves = [node['host'] for node in datanode.nodes()] # if data_changed('namenode.slaves', slaves): # unitdata.kv().set('namenode.slaves', slaves) # hdfs.register_slaves(slaves) # hookenv.status_set('active', 'Ready ({count} DataNode{s})'.format( # count=len(slaves), # s='s' if len(slaves) > 1 else '', # )) set_state('namenode.ready') hookenv.status_set('active', 'ready')
def send_info(nodemanager): hadoop = get_hadoop_base() yarn = YARN(hadoop) local_hostname = hookenv.local_unit().replace('/', '-') port = hadoop.dist_config.port('resourcemanager') hs_http = hadoop.dist_config.port('jh_webapp_http') hs_ipc = hadoop.dist_config.port('jobhistory') utils.update_kv_hosts(nodemanager.hosts_map()) utils.manage_etc_hosts() nodemanager.send_spec(hadoop.spec()) nodemanager.send_resourcemanagers([local_hostname]) nodemanager.send_ports(port, hs_http, hs_ipc) nodemanager.send_ssh_key(utils.get_ssh_key('yarn')) nodemanager.send_hosts_map(utils.get_kv_hosts()) slaves = nodemanager.nodes() if data_changed('resourcemanager.slaves', slaves): unitdata.kv().set('resourcemanager.slaves', slaves) yarn.register_slaves(slaves) hookenv.status_set('active', 'Ready ({count} NodeManager{s})'.format( count=len(slaves), s='s' if len(slaves) > 1 else '', )) set_state('resourcemanager.ready')
def send_nm_all_info(nodemanager): """Send nodemanagers all mapred-slave relation data. At this point, the resourcemanager is ready to serve nodemanagers. Send all mapred-slave relation data so that our 'resourcemanager.ready' state becomes set. """ bigtop = Bigtop() rm_host = get_fqdn() rm_ipc = get_layer_opts().port('resourcemanager') jh_ipc = get_layer_opts().port('jobhistory') jh_http = get_layer_opts().port('jh_webapp_http') nodemanager.send_resourcemanagers([rm_host]) nodemanager.send_spec(bigtop.spec()) nodemanager.send_ports(rm_ipc, jh_http, jh_ipc) # hosts_map and ssh_key are required by the mapred-slave interface to signify # RM's readiness. Send them, even though they are not utilized by bigtop. # NB: update KV hosts with all nodemanagers prior to sending the hosts_map # because mapred-slave gates readiness on a NM's presence in the hosts_map. utils.update_kv_hosts(nodemanager.hosts_map()) nodemanager.send_hosts_map(utils.get_kv_hosts()) nodemanager.send_ssh_key('invalid') # update status with slave count and report ready for hdfs num_slaves = len(nodemanager.nodes()) hookenv.status_set('active', 'ready ({count} nodemanager{s})'.format( count=num_slaves, s='s' if num_slaves > 1 else '', )) set_state('apache-bigtop-resourcemanager.ready')
def accept_clients(clients): hadoop = get_hadoop_base() local_hostname = hookenv.local_unit().replace("/", "-") hdfs_port = hadoop.dist_config.port("namenode") webhdfs_port = hadoop.dist_config.port("nn_webapp_http") clients.send_spec(hadoop.spec()) clients.send_namenodes([local_hostname]) clients.send_ports(hdfs_port, webhdfs_port) clients.send_hosts_map(utils.get_kv_hosts()) clients.send_ready(True)
def accept_clients(clients): hadoop = get_hadoop_base() local_hostname = hookenv.local_unit().replace('/', '-') hdfs_port = hadoop.dist_config.port('namenode') webhdfs_port = hadoop.dist_config.port('nn_webapp_http') clients.send_spec(hadoop.spec()) clients.send_namenodes([local_hostname]) clients.send_ports(hdfs_port, webhdfs_port) clients.send_hosts_map(utils.get_kv_hosts()) clients.send_ready(True)
def accept_clients(clients): hadoop = get_hadoop_base() hdfs_port = hadoop.dist_config.port('namenode') webhdfs_port = hadoop.dist_config.port('nn_webapp_http') clients.send_spec(hadoop.spec()) clients.send_clustername(hookenv.service_name()) clients.send_namenodes(get_cluster_nodes()) clients.send_ports(hdfs_port, webhdfs_port) clients.send_hosts_map(utils.get_kv_hosts()) clients.send_ready(True)
def accept_clients(clients): hadoop = get_hadoop_base() local_hostname = hookenv.local_unit().replace('/', '-') port = hadoop.dist_config.port('resourcemanager') hs_http = hadoop.dist_config.port('jh_webapp_http') hs_ipc = hadoop.dist_config.port('jobhistory') clients.send_spec(hadoop.spec()) clients.send_resourcemanagers([local_hostname]) clients.send_ports(port, hs_http, hs_ipc) clients.send_hosts_map(utils.get_kv_hosts()) clients.send_ready(True)
def provide(self, remote_service, all_ready): data = super(EtcHostsRelation, self).provide(remote_service, all_ready) data.update({ 'etc_hosts': json.dumps(utils.get_kv_hosts()), }) return data
def manage_datanode_hosts(datanode): utils.update_kv_hosts(datanode.hosts_map()) utils.manage_etc_hosts() datanode.send_hosts_map(utils.get_kv_hosts())