def configure_yarn(resourcemanager): hadoop = get_bigtop_base() yarn = YARN(hadoop) utils.update_kv_hosts(resourcemanager.hosts_map()) utils.manage_etc_hosts() if not resourcemanager.resourcemanagers(): data = yaml.dump( { 'relation_name': resourcemanager.relation_name, 'conversations': { conv.key: dict({'relation_ids': conv.relation_ids}, **conv.serialize(conv)) for conv in resourcemanager.conversations() }, 'relation_data': { rid: { unit: hookenv.relation_get(unit=unit, rid=rid) for unit in hookenv.related_units(rid) } for rid in hookenv.relation_ids( resourcemanager.relation_name) }, }, default_flow_style=False) for line in data.splitlines(): hookenv.log(line) yarn.configure_yarn_base(resourcemanager.resourcemanagers()[0], resourcemanager.port(), resourcemanager.hs_http(), resourcemanager.hs_ipc()) set_state('bigtop.yarn.configured')
def configure_hdfs(namenode): hadoop = get_bigtop_base() hdfs = HDFS(hadoop) utils.update_kv_hosts(namenode.hosts_map()) utils.manage_etc_hosts() if not namenode.namenodes(): data = yaml.dump( { 'relation_name': namenode.relation_name, 'conversations': { conv.key: dict({'relation_ids': conv.relation_ids}, **conv.serialize(conv)) for conv in namenode.conversations() }, 'relation_data': { rid: { unit: hookenv.relation_get(unit=unit, rid=rid) for unit in hookenv.related_units(rid) } for rid in hookenv.relation_ids(namenode.relation_name) }, }, default_flow_style=False) for line in data.splitlines(): hookenv.log(line) hdfs.configure_hdfs_base(namenode.namenodes()[0], namenode.port()) set_state('bigtop.hdfs.configured')
def send_info(datanode): hadoop = get_bigtop_base() # hdfs = HDFS(hadoop) # local_hostname = hookenv.local_unit().replace('/', '-') # hdfs_port = hadoop.dist_config.port('namenode') # webhdfs_port = hadoop.dist_config.port('nn_webapp_http') utils.update_kv_hosts({node['ip']: node['host'] for node in datanode.nodes()}) utils.manage_etc_hosts() # datanode.send_spec(hadoop.spec()) # datanode.send_namenodes([local_hostname]) # datanode.send_ports(hdfs_port, webhdfs_port) # datanode.send_ssh_key(utils.get_ssh_key('hdfs')) datanode.send_hosts_map(utils.get_kv_hosts()) # slaves = [node['host'] for node in datanode.nodes()] # if data_changed('namenode.slaves', slaves): # unitdata.kv().set('namenode.slaves', slaves) # hdfs.register_slaves(slaves) # hookenv.status_set('active', 'Ready ({count} DataNode{s})'.format( # count=len(slaves), # s='s' if len(slaves) > 1 else '', # )) set_state('namenode.ready') hookenv.status_set('active', 'ready')
def configure_yarn(resourcemanager): hadoop = get_bigtop_base() yarn = YARN(hadoop) utils.update_kv_hosts(resourcemanager.hosts_map()) utils.manage_etc_hosts() if not resourcemanager.resourcemanagers(): data = yaml.dump({ 'relation_name': resourcemanager.relation_name, 'conversations': { conv.key: dict({'relation_ids': conv.relation_ids}, **conv.serialize(conv)) for conv in resourcemanager.conversations() }, 'relation_data': { rid: { unit: hookenv.relation_get(unit=unit, rid=rid) for unit in hookenv.related_units(rid) } for rid in hookenv.relation_ids( resourcemanager.relation_name ) }, }, default_flow_style=False) for line in data.splitlines(): hookenv.log(line) yarn.configure_yarn_base( resourcemanager.resourcemanagers()[0], resourcemanager.port(), resourcemanager.hs_http(), resourcemanager.hs_ipc()) set_state('bigtop.yarn.configured')
def configure_namenode(): bigtop = get_bigtop_base() # use layer options someday, for now, hard code ports hookenv.open_port('8020') hookenv.open_port('50070') set_state('namenode.started')
def install_hadoop(namenode): '''Install only if the namenode has sent its FQDN.''' if namenode.namenodes(): hookenv.status_set('maintenance', 'installing datanode') hostname = namenode.namenodes()[0] bigtop = get_bigtop_base() bigtop.install(NN=hostname) set_state('datanode.installed') hookenv.status_set('maintenance', 'datanode installed') else: hookenv.status_set('waiting', 'waiting for namenode to become ready')
def install_hadoop(resourcemanager): '''Install only if the resourcemanager has sent its FQDN.''' if resourcemanager.resourcemanagers(): hookenv.status_set('maintenance', 'installing nodemanager') hostname = resourcemanager.resourcemanagers()[0] bigtop = get_bigtop_base() bigtop.install(NN=hostname) set_state('nodemanager.installed') hookenv.status_set('maintenance', 'nodemanager installed') else: hookenv.status_set('waiting', 'waiting for nodemanager to become ready')
def configure_hdfs(namenode): hadoop = get_bigtop_base() hdfs = HDFS(hadoop) utils.update_kv_hosts(namenode.hosts_map()) utils.manage_etc_hosts() if not namenode.namenodes(): data = yaml.dump({ 'relation_name': namenode.relation_name, 'conversations': { conv.key: dict({'relation_ids': conv.relation_ids}, **conv.serialize(conv)) for conv in namenode.conversations() }, 'relation_data': { rid: { unit: hookenv.relation_get(unit=unit, rid=rid) for unit in hookenv.related_units(rid) } for rid in hookenv.relation_ids(namenode.relation_name) }, }, default_flow_style=False) for line in data.splitlines(): hookenv.log(line) hdfs.configure_hdfs_base(namenode.namenodes()[0], namenode.port()) set_state('bigtop.hdfs.configured')
def set_yarn_spec(resourcemanager): hadoop = get_bigtop_base() resourcemanager.set_local_spec(hadoop.spec())
def set_hdfs_spec(namenode): hadoop = get_bigtop_base() namenode.set_local_spec(hadoop.spec())
def install_hadoop(): hadoop = get_bigtop_base() hadoop.install() set_state('bigtop.installed')
def fetch_resources(): hadoop = get_bigtop_base() if hadoop.verify_resources(): set_state('resources.available')
def install_hadoop(): hookenv.status_set('maintenance', 'installing namenode') bigtop = get_bigtop_base() bigtop.install() set_state('namenode.installed') hookenv.status_set('maintenance', 'namenode installed')
def handle_legacy_installed_flag(): hadoop = get_bigtop_base() if hadoop.is_installed(): set_state('bigtop.installed')
def install_hadoop(): bigtop = get_bigtop_base() bigtop.install() set_state('namenode.installed')