def start_cluster(self, cluster): self._set_cluster_info(cluster) deploy.start_cluster(cluster) cluster_instances = plugin_utils.get_instances(cluster) swift_helper.install_ssl_certs(cluster_instances) deploy.add_hadoop_swift_jar(cluster_instances) deploy.prepare_hive(cluster) deploy.deploy_kerberos_principals(cluster)
def configure_swift(self, cluster, instances=None): if self.c_helper.is_swift_enabled(cluster): if not instances: instances = u.get_instances(cluster) u.add_provisioning_step( cluster.id, _("Configure Swift"), len(instances)) with context.PluginsThreadGroup() as tg: for i in instances: tg.spawn('cdh-swift-conf-%s' % i.instance_name, self._configure_swift_to_inst, i) swift_helper.install_ssl_certs(instances)
def scale_cluster(pctx, cluster, instances): config.configure_instances(pctx, instances) _update_include_files(cluster) run.refresh_hadoop_nodes(cluster) rm = vu.get_resourcemanager(cluster) if rm: run.refresh_yarn_nodes(cluster) config.configure_topology_data(pctx, cluster) run.start_dn_nm_processes(instances) swift_helper.install_ssl_certs(instances) config.configure_zookeeper(cluster) run.refresh_zk_servers(cluster)
def scale_cluster(self, cluster, instances): deploy.prepare_kerberos(cluster, instances) deploy.setup_agents(cluster, instances) cluster = conductor.cluster_get(context.ctx(), cluster.id) deploy.wait_host_registration(cluster, instances) deploy.resolve_package_conflicts(cluster, instances) deploy.add_new_hosts(cluster, instances) deploy.manage_config_groups(cluster, instances) deploy.manage_host_components(cluster, instances) deploy.configure_rack_awareness(cluster, instances) swift_helper.install_ssl_certs(instances) deploy.add_hadoop_swift_jar(instances) deploy.deploy_kerberos_principals(cluster, instances)
def scale_cluster(self, cluster, instances): master = utils.get_instance(cluster, "master") r_master = utils.get_remote(master) run.stop_spark(r_master, self._spark_home(cluster)) self._setup_instances(cluster, instances) nn = utils.get_instance(cluster, "namenode") run.refresh_nodes(utils.get_remote(nn), "dfsadmin") dn_instances = [instance for instance in instances if 'datanode' in instance.node_group.node_processes] self._start_datanode_processes(dn_instances) swift_helper.install_ssl_certs(instances) run.start_spark_master(r_master, self._spark_home(cluster)) LOG.info("Spark master service has been restarted")
def start_cluster(self, cluster): keypairs.provision_keypairs(cluster) s_scripts.start_namenode(cluster) s_scripts.start_secondarynamenode(cluster) s_scripts.start_resourcemanager(cluster) run.start_dn_nm_processes(utils.get_instances(cluster)) run.await_datanodes(cluster) s_scripts.start_historyserver(cluster) s_scripts.start_oozie(self.pctx, cluster) s_scripts.start_hiveserver(self.pctx, cluster) s_scripts.start_zookeeper(cluster) swift_helper.install_ssl_certs(utils.get_instances(cluster)) self._set_cluster_info(cluster) s_scripts.start_spark(cluster)
def start_cluster(self, cluster): nn_instance = utils.get_instance(cluster, "namenode") dn_instances = utils.get_instances(cluster, "datanode") # Start the name node self._start_namenode(nn_instance) # start the data nodes self._start_datanode_processes(dn_instances) run.await_datanodes(cluster) LOG.info("Hadoop services have been started") with utils.get_remote(nn_instance) as r: r.execute_command("sudo -u hdfs hdfs dfs -mkdir -p /user/$USER/") r.execute_command("sudo -u hdfs hdfs dfs -chown $USER " "/user/$USER/") # start spark nodes self.start_spark(cluster) swift_helper.install_ssl_certs(utils.get_instances(cluster)) LOG.info('Cluster has been started successfully') self._set_cluster_info(cluster)