def install_resourcemanager(namenode): """Install if the namenode has sent its FQDN. We only need the namenode FQDN to perform the RM install, so poll for namenodes() data whenever we have a namenode relation. This allows us to install asap, even if 'namenode.ready' is not set yet. """ if namenode.namenodes(): hookenv.status_set('maintenance', 'installing resourcemanager') nn_host = namenode.namenodes()[0] rm_host = get_fqdn() bigtop = Bigtop() hosts = {'namenode': nn_host, 'resourcemanager': rm_host} bigtop.render_site_yaml(hosts=hosts, roles='resourcemanager') bigtop.trigger_puppet() # /etc/hosts entries from the KV are not currently used for bigtop, # but a hosts_map attribute is required by some interfaces (eg: mapred-slave) # to signify RM's readiness. Set our RM info in the KV to fulfill this # requirement. utils.initialize_kv_host() # Add our ubuntu user to the hadoop and mapred groups. get_layer_opts().add_users() set_state('apache-bigtop-resourcemanager.installed') hookenv.status_set('maintenance', 'resourcemanager installed') else: hookenv.status_set('waiting', 'waiting for namenode fqdn')
def __init__(self, spec=None, port=None, historyserver_http=None, historyserver_ipc=None, *args, **kwargs): self.port = port # only needed for provides self.historyserver_http = historyserver_http # only needed for provides self.historyserver_ipc = historyserver_ipc # only needed for provides utils.initialize_kv_host() super(ResourceManager, self).__init__(spec, *args, **kwargs)
def install_namenode(): hookenv.status_set('maintenance', 'installing namenode') bigtop = Bigtop() nn_host = get_fqdn() hosts = {'namenode': nn_host} bigtop.render_site_yaml(hosts=hosts, roles='namenode') bigtop.trigger_puppet() # /etc/hosts entries from the KV are not currently used for bigtop, # but a hosts_map attribute is required by some interfaces (eg: dfs-slave) # to signify NN's readiness. Set our NN info in the KV to fulfill this # requirement. utils.initialize_kv_host() # make our namenode listen on all interfaces hdfs_site = Path('/etc/hadoop/conf/hdfs-site.xml') with utils.xmlpropmap_edit_in_place(hdfs_site) as props: props['dfs.namenode.rpc-bind-host'] = '0.0.0.0' props['dfs.namenode.servicerpc-bind-host'] = '0.0.0.0' props['dfs.namenode.http-bind-host'] = '0.0.0.0' props['dfs.namenode.https-bind-host'] = '0.0.0.0' # We need to create the 'mapred' user/group since we are not installing # hadoop-mapreduce. This is needed so the namenode can access yarn # job history files in hdfs. Also add our ubuntu user to the hadoop # and mapred groups. get_layer_opts().add_users() set_state('apache-bigtop-namenode.installed') hookenv.status_set('maintenance', 'namenode installed')
def __init__(self, spec=None, port=None, webhdfs_port=None, *args, **kwargs): self.port = port # only needed for provides self.webhdfs_port = webhdfs_port # only needed for provides utils.initialize_kv_host() super(NameNode, self).__init__(spec, *args, **kwargs)
def configure_namenode(): hadoop = get_hadoop_base() hdfs = HDFS(hadoop) hdfs.configure_namenode(get_cluster_nodes()) hdfs.format_namenode() hdfs.start_namenode() hdfs.create_hdfs_dirs() hadoop.open_ports('namenode') utils.initialize_kv_host() utils.manage_etc_hosts() set_state('namenode.started')
def install(self): version = hookenv.config()['spark_version'] spark_path = self.extract_spark_binary('spark-{}'.format(version), version) os.symlink(spark_path, self.dist_config.path('spark')) unitdata.kv().set('spark.version', version) self.dist_config.add_users() self.dist_config.add_dirs() self.dist_config.add_packages() # allow ubuntu user to ssh to itself so spark can ssh to its worker # in local/standalone modes utils.install_ssh_key('ubuntu', utils.get_ssh_key('ubuntu')) utils.initialize_kv_host() utils.manage_etc_hosts() hostname = hookenv.local_unit().replace('/', '-') etc_hostname = Path('/etc/hostname') etc_hostname.write_text(hostname) check_call(['hostname', '-F', etc_hostname]) unitdata.kv().set('spark.installed', True) unitdata.kv().flush(True)
def install_namenode(): hookenv.status_set('maintenance', 'installing namenode') bigtop = Bigtop() hdfs_port = get_layer_opts().port('namenode') webhdfs_port = get_layer_opts().port('nn_webapp_http') bigtop.render_site_yaml( hosts={ 'namenode': get_fqdn(), }, roles=[ 'namenode', 'mapred-app', ], # NB: We want the NN to listen on all interfaces, so bind to 0.0.0.0. overrides={ 'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port, 'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0', 'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port, 'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0', 'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0', } ) bigtop.trigger_puppet() # /etc/hosts entries from the KV are not currently used for bigtop, # but a hosts_map attribute is required by some interfaces (eg: dfs-slave) # to signify NN's readiness. Set our NN info in the KV to fulfill this # requirement. utils.initialize_kv_host() # We need to create the 'mapred' and 'spark' user/group since we may not # be installing hadoop-mapreduce or spark on this machine. This is needed # so the namenode can access yarn and spark job history files in hdfs. Also # add our ubuntu user to the hadoop, mapred, and spark groups. get_layer_opts().add_users() set_state('apache-bigtop-namenode.installed') hookenv.status_set('maintenance', 'namenode installed')
def install_namenode(): hookenv.status_set('maintenance', 'installing namenode') bigtop = Bigtop() hdfs_port = get_layer_opts().port('namenode') webhdfs_port = get_layer_opts().port('nn_webapp_http') bigtop.render_site_yaml( hosts={ 'namenode': get_fqdn(), }, roles=[ 'namenode', 'mapred-app', ], # NB: We want the NN to listen on all interfaces, so bind to 0.0.0.0. overrides={ 'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port, 'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0', 'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port, 'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0', 'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0', }) bigtop.trigger_puppet() # /etc/hosts entries from the KV are not currently used for bigtop, # but a hosts_map attribute is required by some interfaces (eg: dfs-slave) # to signify NN's readiness. Set our NN info in the KV to fulfill this # requirement. utils.initialize_kv_host() # We need to create the 'mapred' user/group since we are not installing # hadoop-mapreduce. This is needed so the namenode can access yarn # job history files in hdfs. Also add our ubuntu user to the hadoop # and mapred groups. get_layer_opts().add_users() set_state('apache-bigtop-namenode.installed') hookenv.status_set('maintenance', 'namenode installed')
def install_namenode(): hookenv.status_set('maintenance', 'installing namenode') bigtop = Bigtop() bigtop.render_site_yaml( hosts={ 'namenode': get_fqdn(), }, roles=[ 'namenode', 'mapred-app', ], ) bigtop.trigger_puppet() # /etc/hosts entries from the KV are not currently used for bigtop, # but a hosts_map attribute is required by some interfaces (eg: dfs-slave) # to signify NN's readiness. Set our NN info in the KV to fulfill this # requirement. utils.initialize_kv_host() # make our namenode listen on all interfaces hdfs_site = Path('/etc/hadoop/conf/hdfs-site.xml') with utils.xmlpropmap_edit_in_place(hdfs_site) as props: props['dfs.namenode.rpc-bind-host'] = '0.0.0.0' props['dfs.namenode.servicerpc-bind-host'] = '0.0.0.0' props['dfs.namenode.http-bind-host'] = '0.0.0.0' props['dfs.namenode.https-bind-host'] = '0.0.0.0' # We need to create the 'mapred' user/group since we are not installing # hadoop-mapreduce. This is needed so the namenode can access yarn # job history files in hdfs. Also add our ubuntu user to the hadoop # and mapred groups. get_layer_opts().add_users() set_state('apache-bigtop-namenode.installed') hookenv.status_set('maintenance', 'namenode installed')
def install_resourcemanager(namenode): """Install if the namenode has sent its FQDN. We only need the namenode FQDN to perform the RM install, so poll for namenodes() data whenever we have a namenode relation. This allows us to install asap, even if 'namenode.ready' is not set yet. """ if namenode.namenodes(): hookenv.status_set('maintenance', 'installing resourcemanager') # Hosts nn_host = namenode.namenodes()[0] rm_host = get_fqdn() # Ports rm_ipc = get_layer_opts().port('resourcemanager') rm_http = get_layer_opts().port('rm_webapp_http') jh_ipc = get_layer_opts().port('jobhistory') jh_http = get_layer_opts().port('jh_webapp_http') hdfs_port = namenode.port() webhdfs_port = namenode.webhdfs_port() bigtop = Bigtop() bigtop.render_site_yaml( hosts={ 'namenode': nn_host, 'resourcemanager': rm_host, }, roles=[ 'resourcemanager', ], # NB: When we colocate the NN and RM, the RM will run puppet apply # last. To ensure we don't lose any hdfs-site.xml data set by the # NN, override common_hdfs properties again here. overrides={ 'hadoop::common_yarn::hadoop_rm_port': rm_ipc, 'hadoop::common_yarn::hadoop_rm_webapp_port': rm_http, 'hadoop::common_yarn::hadoop_rm_bind_host': '0.0.0.0', 'hadoop::common_mapred_app::mapreduce_jobhistory_host': '0.0.0.0', 'hadoop::common_mapred_app::mapreduce_jobhistory_port': jh_ipc, 'hadoop::common_mapred_app::mapreduce_jobhistory_webapp_port': jh_http, 'hadoop::common_hdfs::hadoop_namenode_port': hdfs_port, 'hadoop::common_hdfs::hadoop_namenode_bind_host': '0.0.0.0', 'hadoop::common_hdfs::hadoop_namenode_http_port': webhdfs_port, 'hadoop::common_hdfs::hadoop_namenode_http_bind_host': '0.0.0.0', 'hadoop::common_hdfs::hadoop_namenode_https_bind_host': '0.0.0.0', } ) bigtop.trigger_puppet() # /etc/hosts entries from the KV are not currently used for bigtop, # but a hosts_map attribute is required by some interfaces (eg: mapred-slave) # to signify RM's readiness. Set our RM info in the KV to fulfill this # requirement. utils.initialize_kv_host() # We need to create the 'spark' user/group since we may not be # installing spark on this machine. This is needed so the history # server can access spark job history files in hdfs. Also add our # ubuntu user to the hadoop, mapred, and spark groups on this machine. get_layer_opts().add_users() set_state('apache-bigtop-resourcemanager.installed') hookenv.status_set('maintenance', 'resourcemanager installed') else: hookenv.status_set('waiting', 'waiting for namenode fqdn')