def start_cluster(self, cluster):
self._set_cluster_info(cluster)
deploy.start_cluster(cluster)
cluster_instances = plugin_utils.get_instances(cluster)
swift_helper.install_ssl_certs(cluster_instances)
deploy.add_hadoop_swift_jar(cluster_instances)
deploy.prepare_hive(cluster)
def start_cluster(self, cluster):
keypairs.provision_keypairs(cluster)
des = cluster.description
backup = None
if len(des) > 5 and des[-6:] == "backup":
backup = "backup"
# 配置环境变量
s_scripts.config_user_env(cluster)
s_scripts.start_namenode(cluster, backup)
s_scripts.start_secondarynamenode(cluster)
s_scripts.start_resourcemanager(cluster)
run.start_dn_nm_processes(utils.get_instances(cluster))
run.await_datanodes(cluster)
s_scripts.start_historyserver(cluster)
s_scripts.start_oozie(self.pctx, cluster, backup)
s_scripts.start_hiveserver(self.pctx, cluster, backup)
# swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
# start spark nodes, newly added 'swift_helper.install_ssl_certs' after 'start_spark'
s_scripts.start_spark(cluster)
swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
# 新增的方法，配置用户的环境变量
# s_scripts.config_user_env(cluster)
# newly added
LOG.info('Cluster has been started successfully --by HXH')
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
self._set_cluster_info(cluster)
deploy.start_cluster(cluster)
cluster_instances = plugin_utils.get_instances(cluster)
swift_helper.install_ssl_certs(cluster_instances)
deploy.add_hadoop_swift_jar(cluster_instances)
deploy.prepare_hive(cluster)
deploy.deploy_kerberos_principals(cluster)
def scale_cluster(self, cluster, instances):
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
swift_helper.install_ssl_certs(instances)
def scale_cluster(self, cluster, instances):
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
swift_helper.install_ssl_certs(instances)
def configure_swift(self, cluster, instances=None):
if self.c_helper.is_swift_enabled(cluster):
if not instances:
instances = u.get_instances(cluster)
cpo.add_provisioning_step(cluster.id, _("Configure Swift"), len(instances))
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn("cdh-swift-conf-%s" % i.instance_name, self._configure_swift_to_inst, i)
swift_helper.install_ssl_certs(instances)
def scale_cluster(pctx, cluster, instances):
config.configure_instances(pctx, instances)
_update_include_files(cluster)
run.refresh_hadoop_nodes(cluster)
rm = vu.get_resourcemanager(cluster)
if rm:
run.refresh_yarn_nodes(cluster)
config.configure_topology_data(pctx, cluster)
run.start_dn_nm_processes(instances)
swift_helper.install_ssl_certs(instances)
def scale_cluster(pctx, cluster, instances):
config.configure_instances(pctx, instances)
_update_include_files(cluster)
run.refresh_hadoop_nodes(cluster)
rm = vu.get_resourcemanager(cluster)
if rm:
run.refresh_yarn_nodes(cluster)
config.configure_topology_data(pctx, cluster)
run.start_dn_nm_processes(instances)
swift_helper.install_ssl_certs(instances)
def configure_swift(self, cluster, instances=None):
if self.c_helper.is_swift_enabled(cluster):
if not instances:
instances = u.get_instances(cluster)
cpo.add_provisioning_step(cluster.id, _("Configure Swift"),
len(instances))
with context.ThreadGroup() as tg:
for i in instances:
tg.spawn('cdh-swift-conf-%s' % i.instance_name,
self._configure_swift_to_inst, i)
swift_helper.install_ssl_certs(instances)
def scale_cluster(self, cluster, instances):
deploy.prepare_kerberos(cluster, instances)
deploy.setup_agents(cluster, instances)
cluster = conductor.cluster_get(context.ctx(), cluster.id)
deploy.wait_host_registration(cluster, instances)
deploy.resolve_package_conflicts(cluster, instances)
deploy.add_new_hosts(cluster, instances)
deploy.manage_config_groups(cluster, instances)
deploy.manage_host_components(cluster, instances)
deploy.configure_rack_awareness(cluster, instances)
swift_helper.install_ssl_certs(instances)
deploy.add_hadoop_swift_jar(instances)
deploy.deploy_kerberos_principals(cluster, instances)
def start_cluster(self, cluster):
s_scripts.start_namenode(cluster)
s_scripts.start_secondarynamenode(cluster)
s_scripts.start_resourcemanager(cluster)
run.start_dn_nm_processes(utils.get_instances(cluster))
run.await_datanodes(cluster)
s_scripts.start_historyserver(cluster)
s_scripts.start_oozie(self.pctx, cluster)
s_scripts.start_hiveserver(self.pctx, cluster)
swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
self._set_cluster_info(cluster)
def scale_cluster(self, cluster, instances):
master = utils.get_instance(cluster, "master")
r_master = remote.get_remote(master)
run.stop_spark(r_master, self._spark_home(cluster))
self._setup_instances(cluster, instances)
nn = utils.get_instance(cluster, "namenode")
run.refresh_nodes(remote.get_remote(nn), "dfsadmin")
dn_instances = [instance for instance in instances if "datanode" in instance.node_group.node_processes]
self._start_datanode_processes(dn_instances)
swift_helper.install_ssl_certs(instances)
run.start_spark_master(r_master, self._spark_home(cluster))
LOG.info(_LI("Spark master service has been restarted"))
def start_cluster(self, cluster):
s_scripts.start_namenode(cluster)
s_scripts.start_secondarynamenode(cluster)
s_scripts.start_resourcemanager(cluster)
run.start_dn_nm_processes(utils.get_instances(cluster))
run.await_datanodes(cluster)
s_scripts.start_historyserver(cluster)
s_scripts.start_oozie(self.pctx, cluster)
s_scripts.start_hiveserver(self.pctx, cluster)
swift_helper.install_ssl_certs(cluster_utils.get_instances(cluster))
self._set_cluster_info(cluster)
def scale_cluster(self, cluster, instances):
master = utils.get_instance(cluster, "master")
r_master = remote.get_remote(master)
run.stop_spark(r_master, self._spark_home(cluster))
self._setup_instances(cluster, instances)
nn = utils.get_instance(cluster, "namenode")
run.refresh_nodes(remote.get_remote(nn), "dfsadmin")
dn_instances = [instance for instance in instances if
'datanode' in instance.node_group.node_processes]
self._start_datanode_processes(dn_instances)
swift_helper.install_ssl_certs(instances)
run.start_spark_master(r_master, self._spark_home(cluster))
LOG.info("Spark master service has been restarted")
def start_cluster(self, cluster):
nn_instance = utils.get_instance(cluster, "namenode")
dn_instances = utils.get_instances(cluster, "datanode")
# Start the name node
self._start_namenode(nn_instance)
# start the data nodes
self._start_datanode_processes(dn_instances)
LOG.info(_LI("Hadoop services have been started"))
with remote.get_remote(nn_instance) as r:
r.execute_command("sudo -u hdfs hdfs dfs -mkdir -p /user/$USER/")
r.execute_command("sudo -u hdfs hdfs dfs -chown $USER " "/user/$USER/")
# start spark nodes
self.start_spark(cluster)
swift_helper.install_ssl_certs(utils.get_instances(cluster))
LOG.info(_LI("Cluster has been started successfully"))
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
nn_instance = utils.get_instance(cluster, "namenode")
dn_instances = utils.get_instances(cluster, "datanode")
# Start the name node
self._start_namenode(nn_instance)
# start the data nodes
self._start_datanode_processes(dn_instances)
LOG.info(_LI("Hadoop services have been started"))
with remote.get_remote(nn_instance) as r:
r.execute_command("sudo -u hdfs hdfs dfs -mkdir -p /user/$USER/")
r.execute_command("sudo -u hdfs hdfs dfs -chown $USER "
"/user/$USER/")
# start spark nodes
self.start_spark(cluster)
swift_helper.install_ssl_certs(utils.get_instances(cluster))
LOG.info(_LI('Cluster has been started successfully'))
self._set_cluster_info(cluster)
def start_cluster(self, cluster):
self._set_cluster_info(cluster)
deploy.start_cluster(cluster)
swift_helper.install_ssl_certs(plugin_utils.get_instances(cluster))
def start_cluster(self, cluster):
self._set_cluster_info(cluster)
deploy.start_cluster(cluster)
cluster_instances = plugin_utils.get_instances(cluster)
swift_helper.install_ssl_certs(cluster_instances)
deploy.add_hadoop_swift_jar(cluster_instances)
def install_ssl_certs(instances, **kwargs):
swift_helper.install_ssl_certs(instances)
def start_cluster(self, cluster):
self._set_cluster_info(cluster)
deploy.start_cluster(cluster)
swift_helper.install_ssl_certs(plugin_utils.get_instances(cluster))