def start_cluster(self, cluster): nn_instance = utils.get_namenode(cluster) datanodes = utils.get_datanodes(cluster) jt_instance = utils.get_jobtracker(cluster) tasktrackers = utils.get_tasktrackers(cluster) oozie = utils.get_oozie(cluster) with remote.get_remote(nn_instance) as r: run.format_namenode(r) run.start_process(r, "namenode") snns = utils.get_secondarynamenodes(cluster) if snns: for snn in snns: run.start_process(remote.get_remote(snn), "secondarynamenode") for dn in datanodes: run.start_process(remote.get_remote(dn), "datanode") LOG.info("HDFS service at '%s' has been started", nn_instance.hostname) if jt_instance: run.start_process(remote.get_remote(jt_instance), "jobtracker") for tt in tasktrackers: run.start_process(remote.get_remote(tt), "tasktracker") LOG.info("MapReduce service at '%s' has been started", jt_instance.hostname) if oozie: with remote.get_remote(oozie) as r: run.oozie_share_lib(r, nn_instance.hostname) run.start_oozie(r) LOG.info("Oozie service at '%s' has been started", nn_instance.hostname) LOG.info('Cluster %s has been started successfully' % cluster.name) self._set_cluster_info(cluster)
def start_cluster(self, cluster): nn_instance = utils.get_namenode(cluster) datanodes = utils.get_datanodes(cluster) jt_instance = utils.get_jobtracker(cluster) tasktrackers = utils.get_tasktrackers(cluster) with nn_instance.remote as remote: run.format_namenode(remote) run.start_process(remote, "namenode") snns = utils.get_secondarynamenodes(cluster) if snns: for snn in snns: run.start_process(snn.remote, "secondarynamenode") for dn in datanodes: run.start_process(dn.remote, "datanode") LOG.info("HDFS service at '%s' has been started", nn_instance.hostname) if jt_instance: run.start_process(jt_instance.remote, "jobtracker") for tt in tasktrackers: run.start_process(tt.remote, "tasktracker") LOG.info("MapReduce service at '%s' has been started", jt_instance.hostname) LOG.info('Cluster %s has been started successfully' % cluster.name) self._set_cluster_info(cluster)
def start_cluster(self, cluster): nn_instance = utils.get_namenode(cluster) datanodes = utils.get_datanodes(cluster) jt_instance = utils.get_jobtracker(cluster) tasktrackers = utils.get_tasktrackers(cluster) oozie = utils.get_oozie(cluster) hive_server = utils.get_hiveserver(cluster) with remote.get_remote(nn_instance) as r: run.format_namenode(r) run.start_process(r, "namenode") snns = utils.get_secondarynamenodes(cluster) if snns: for snn in snns: run.start_process(remote.get_remote(snn), "secondarynamenode") for dn in datanodes: run.start_process(remote.get_remote(dn), "datanode") LOG.info("HDFS service at '%s' has been started", nn_instance.hostname) if jt_instance: run.start_process(remote.get_remote(jt_instance), "jobtracker") for tt in tasktrackers: run.start_process(remote.get_remote(tt), "tasktracker") LOG.info("MapReduce service at '%s' has been started", jt_instance.hostname) if oozie: with remote.get_remote(oozie) as r: if c_helper.is_mysql_enable(cluster): run.mysql_start(r, oozie) run.oozie_create_db(r) run.oozie_share_lib(r, nn_instance.hostname) run.start_oozie(r) LOG.info("Oozie service at '%s' has been started", nn_instance.hostname) if hive_server: with remote.get_remote(nn_instance) as r: run.hive_create_warehouse_dir(r) if c_helper.is_mysql_enable(cluster): with remote.get_remote(hive_server) as h: if not oozie or hive_server.hostname != oozie.hostname: run.mysql_start(h, hive_server) run.hive_create_db(h) run.hive_metastore_start(h) LOG.info("Hive Metastore server at %s has been started", hive_server.hostname) LOG.info('Cluster %s has been started successfully' % cluster.name) self._set_cluster_info(cluster)
def scale_cluster(self, cluster, instances): self._push_configs_to_nodes(cluster, instances=instances) self._write_hadoop_user_keys(cluster.private_key, instances) run.refresh_nodes(remote.get_remote(utils.get_namenode(cluster)), "dfsadmin") jt = utils.get_jobtracker(cluster) if jt: run.refresh_nodes(remote.get_remote(jt), "mradmin") for i in instances: with remote.get_remote(i) as r: if "datanode" in i.node_group.node_processes: run.start_process(r, "datanode") if "tasktracker" in i.node_group.node_processes: run.start_process(r, "tasktracker")
def scale_cluster(self, cluster, instances): self._extract_configs(cluster) self._push_configs_to_nodes(cluster, instances=instances) self._write_hadoop_user_keys(cluster.private_key, instances) run.refresh_nodes(utils.get_namenode(cluster).remote, "dfsadmin") jt = utils.get_jobtracker(cluster) if jt: run.refresh_nodes(jt.remote, "mradmin") for i in instances: with i.remote as remote: if "datanode" in i.node_group.node_processes: run.start_process(remote, "datanode") if "tasktracker" in i.node_group.node_processes: run.start_process(remote, "tasktracker")
def scale_cluster(self, cluster, instances): self._push_configs_to_nodes(cluster, instances=instances) self._write_hadoop_user_keys(instances, cluster.management_private_key, cluster.management_public_key) run.refresh_nodes(remote.get_remote( utils.get_namenode(cluster)), "dfsadmin") jt = utils.get_jobtracker(cluster) if jt: run.refresh_nodes(remote.get_remote(jt), "mradmin") for i in instances: with remote.get_remote(i) as r: if "datanode" in i.node_group.node_processes: run.start_process(r, "datanode") if "tasktracker" in i.node_group.node_processes: run.start_process(r, "tasktracker")
def start_cluster(self, cluster): nn_instance = utils.get_namenode(cluster) datanodes = utils.get_datanodes(cluster) jt_instance = utils.get_jobtracker(cluster) tasktrackers = utils.get_tasktrackers(cluster) oozie = utils.get_oozie(cluster) with remote.get_remote(nn_instance) as r: run.format_namenode(r) run.start_process(r, "namenode") snns = utils.get_secondarynamenodes(cluster) if snns: for snn in snns: run.start_process(remote.get_remote(snn), "secondarynamenode") for dn in datanodes: run.start_process(remote.get_remote(dn), "datanode") LOG.info("HDFS service at '%s' has been started", nn_instance.hostname) if jt_instance: run.start_process(remote.get_remote(jt_instance), "jobtracker") for tt in tasktrackers: run.start_process(remote.get_remote(tt), "tasktracker") LOG.info("MapReduce service at '%s' has been started", jt_instance.hostname) if oozie: with remote.get_remote(oozie) as r: run.oozie_share_lib(r, nn_instance.hostname) run.start_oozie(r) LOG.info("Oozie service at '%s' has been started", nn_instance.hostname) LOG.info('Cluster %s has been started successfully' % cluster.name) self._set_cluster_info(cluster)