def job_executer_add_work_node(self):
self.logger.info("{0} job begins !".format(self.jobname))
# sftp your script to remote host with paramiko.
srcipt_package = "{0}.tar".format(self.jobname)
src_local = "parcel-center/{0}".format(self.node_config["nodename"])
dst_remote = common.get_user_dir(self.node_config)
if common.sftp_paramiko(src_local, dst_remote, srcipt_package, self.node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}.tar".format(self.jobname, self.node_config['hostip'])
if common.ssh_shell_paramiko(self.node_config, commandline) == False:
self.logger.error("Failed to uncompress {0}.tar".format(self.jobname))
sys.exit(1)
commandline = "sudo ./{0}/hosts-check.sh {1}".format(self.jobname, self.node_config['hostip'])
if common.ssh_shell_with_password_input_paramiko(self.node_config, commandline) == False:
self.logger.error("Failed to update the /etc/hosts on {0}".format(self.node_config['hostip']))
sys.exit(1)
commandline = "sudo ./{0}/docker-ce-install.sh".format(self.jobname)
if common.ssh_shell_with_password_input_paramiko(self.node_config, commandline) == False:
self.logger.error("Failed to install docker-ce on {0}".format(self.node_config['hostip']))
sys.exit(1)
commandline = "sudo ./{0}/kubelet-start.sh {0}".format(self.jobname)
if common.ssh_shell_with_password_input_paramiko(self.node_config, commandline) == False:
self.logger.error("Failed to bootstrap kubelet on {0}".format(self.node_config['hostip']))
sys.exit(1)
self.logger.info("Successfully running {0} job on node {1}".format(self.jobname, self.node_config["nodename"]))
def job_executer_starting_new_master_node(self):
com = self.cluster_object_model
new_etcd_cluster_ips_peer = pai_common.get_new_etcd_peer_ip_list(
com, self.node_config)
com['kubernetes']['etcd_cluster_ips_peer'] = new_etcd_cluster_ips_peer
com['kubernetes']['etcd-initial-cluster-state'] = 'existing'
self.logger.info("---- package wrapper is working now! ----")
self.prepare_package(self.node_config, "add-master-node-task-two")
self.logger.info("---- package wrapper's work finished ----")
script_package = "add-master-node-task-two.tar"
src_local = "parcel-center/{0}".format(self.node_config["nodename"])
dst_remote = common.get_user_dir(self.node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
self.node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}".format(script_package)
if common.ssh_shell_with_password_input_paramiko(
self.node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully extract the script package for add-master-node-task-two!"
)
commandline = "sudo ./{0}/hosts-check.sh {1}".format(
"add-master-node-task-two", self.node_config['hostip'])
if common.ssh_shell_with_password_input_paramiko(
self.node_config, commandline) == False:
self.logger.error("Failed to update the /etc/hosts on {0}".format(
self.node_config['hostip']))
sys.exit(1)
commandline = "sudo ./{0}/docker-ce-install.sh {0}".format(
"add-master-node-task-two")
if common.ssh_shell_with_password_input_paramiko(
self.node_config, commandline) == False:
self.logger.error("Failed to install docker-ce on {0}".format(
self.node_config['hostip']))
sys.exit(1)
commandline = "sudo ./{0}/kubelet-start.sh {0}".format(
"add-master-node-task-two")
if common.ssh_shell_with_password_input_paramiko(
self.node_config, commandline) == False:
self.logger.error("Failed to bootstrap kubelet on {0}".format(
self.node_config['hostip']))
sys.exit(1)
self.logger.info("Successfully running {0} job on node {1}!".format(
"add-master-node-task-two", self.node_config['hostip']))
if self.clean_flag:
self.delete_packege(self.node_config)
self.remote_host_cleaner(self.node_config,
"add-master-node-task-two")
def job_execute_remove_node_from_etcd_cluster(self):
# Waiting for the bad node to remove from leader.
while True:
leader_node_config = pai_common.get_etcd_leader_node(
self.cluster_config)
if leader_node_config == None:
self.logger.error(
"Failed to find the leader node in the etcd cluster")
sys.exit(1)
if leader_node_config['nodename'] != self.node_config['nodename']:
break
self.prepare_package(leader_node_config,
"remove-node-from-etcd-cluster")
self.logger.info(
"Begin to execute the job : remove-node-from-etcd-cluster.")
self.logger.info("Update etcd cluster on host [{0}].".format(
leader_node_config['nodename']))
script_package = "remove-node-from-etcd-cluster.tar"
src_local = "parcel-center/{0}".format(leader_node_config["nodename"])
dst_remote = common.get_user_dir(leader_node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
leader_node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}.tar".format(
"remove-node-from-etcd-cluster")
if common.ssh_shell_with_password_input_paramiko(
leader_node_config, commandline) == False:
sys.exit(1)
commandline = "sudo /bin/bash {0}/{1}.sh {2} {3}".format(
"remove-node-from-etcd-cluster", "remove-member-from-etcd-cluster",
self.node_config['hostip'], self.node_config['etcdid'])
if common.ssh_shell_with_password_input_paramiko(
leader_node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully remove target node from etcd cluster on node {0}".
format(leader_node_config["nodename"]))
if self.clean_flag == True:
self.logger.info("---- package cleaner is working now! ----")
self.delete_packege(leader_node_config)
self.logger.info("---- package cleaner's work finished! ----")
self.logger.info("---- remote host cleaner is working now! ----")
self.remote_host_cleaner(leader_node_config,
"remove-node-from-etcd-cluster")
self.logger.info("---- remote host cleaning job finished! ")
def update_etcd_cluster(self, good_node_config, bad_node_config):
self.prepare_package(good_node_config, "etcd-reconfiguration-update")
self.logger.info(
"Begin to execute the job : etcd-reconfiguration-update.")
self.logger.info("Update etcd cluster on host [{0}].".format(
good_node_config['nodename']))
script_package = "etcd-reconfiguration-update.tar"
src_local = "parcel-center/{0}".format(good_node_config["nodename"])
dst_remote = common.get_user_dir(good_node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
good_node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}.tar".format("etcd-reconfiguration-update")
if common.ssh_shell_with_password_input_paramiko(
good_node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully extract the script package for etcd-reconfiguration-update!"
)
commandline = "sudo /bin/bash {0}/{1}.sh {2} {3}".format(
"etcd-reconfiguration-update", "remove-member-from-etcd-cluster",
bad_node_config['hostip'], bad_node_config['etcdid'])
if common.ssh_shell_with_password_input_paramiko(
good_node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully remove the bad-member from the etcd cluster.")
commandline = "sudo /bin/bash {0}/{1}.sh {2} {3}".format(
"etcd-reconfiguration-update", "add-member-to-etcd-cluster",
bad_node_config['hostip'], bad_node_config['etcdid'])
if common.ssh_shell_with_password_input_paramiko(
good_node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully add the bad-member into the etcd cluster again.")
self.logger.info(
"Successfully update etcd cluster configuration on node {0}".
format(bad_node_config["nodename"]))
if self.clean_flag:
self.delete_packege(good_node_config)
def remote_host_cleaner(self, node_config, jobname):
commandline = "sudo rm -rf {0}*".format(jobname)
if common.ssh_shell_with_password_input_paramiko(
node_config, commandline) == False:
sys.exit(1)
def update_etcd_cluster(self, good_node_config, bad_node_config):
self.prepare_package(good_node_config, "etcd-reconfiguration-update")
self.logger.info(
"Begin to execute the job : etcd-reconfiguration-update.")
self.logger.info("Update etcd cluster on host [{0}].".format(
good_node_config['nodename']))
script_package = "etcd-reconfiguration-update.tar"
src_local = "parcel-center/{0}".format(good_node_config["nodename"])
dst_remote = common.get_user_dir(good_node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
good_node_config) == False:
return
commandline = "tar -xvf {0}.tar && sudo ./{0}/{1}.sh {2} {3}".format(
"etcd-reconfiguration-update", "update-etcd-cluster",
bad_node_config['hostip'], bad_node_config['etcdid'])
if common.ssh_shell_with_password_input_paramiko(
good_node_config, commandline) == False:
return
self.logger.info(
"Successfully update etcd cluster configuration on node {0}".
format(bad_node_config["nodename"]))
if self.clean_flag:
self.delete_packege(good_node_config)
def stop_bad_etcd_server(self, bad_node_config):
self.prepare_package(bad_node_config, "etcd-reconfiguration-stop")
self.logger.info(
"Begin to execute the job : etcd-reconfiguration-stop.")
self.logger.info("Stop the bad etcd server on host [{0}]".format(
bad_node_config['nodename']))
script_package = "etcd-reconfiguration-stop.tar"
src_local = "parcel-center/{0}".format(bad_node_config["nodename"])
dst_remote = common.get_user_dir(bad_node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
bad_node_config) == False:
return
commandline = "tar -xvf {0}.tar && sudo ./{0}/stop-etcd-server.sh".format(
"etcd-reconfiguration-stop")
if common.ssh_shell_with_password_input_paramiko(
bad_node_config, commandline) == False:
return
self.logger.info(
"Successfully stoping bad etcd server on node {0}".format(
bad_node_config["nodename"]))
if self.clean_flag:
self.delete_packege(bad_node_config)
def job_executer(self, node_config):
self.logger.info("{0} job begins !".format(self.jobname))
# sftp your script to remote host with paramiko.
srcipt_package = "{0}.tar".format(self.jobname)
src_local = "parcel-center/{0}".format(node_config["nodename"])
dst_remote = common.get_user_dir(node_config)
if common.sftp_paramiko(src_local, dst_remote, srcipt_package,
node_config) == False:
return
commandline = "tar -xvf {0}.tar".format(self.jobname,
node_config['hostip'])
if common.ssh_shell_paramiko(node_config, commandline) == False:
self.logger.error("Failed to uncompress {0}.tar".format(
self.jobname))
return
commandline = "sudo ./{0}/kubernetes-cleanup.sh".format(self.jobname)
if common.ssh_shell_with_password_input_paramiko(
node_config, commandline) == False:
self.logger.error(
"Failed to cleanup the kubernetes deployment on {0}".format(
node_config['hostip']))
return
self.logger.info("Successfully running {0} job on node {1}".format(
self.jobname, node_config["nodename"]))
def remote_host_cleaner(self):
commandline = "sudo rm -rf {0}*".format(self.jobname)
if common.ssh_shell_with_password_input_paramiko(
self.node_config, commandline) == False:
return
def restart_etcd_server(self, bad_node_config):
self.logger.info("Begin to execute the job : etcd-reconfiguration-restart.")
self.logger.info("Restart etcd server on host [{0}].".format(bad_node_config['nodename']))
new_etcd_cluster_ips_peer = self.get_etcd_peer_ip_list(bad_node_config)
self.cluster_config['clusterinfo']['etcd_cluster_ips_peer'] = new_etcd_cluster_ips_peer
self.cluster_config['clusterinfo']['etcd-initial-cluster-state'] = 'existing'
self.prepare_package(bad_node_config, "etcd-reconfiguration-restart")
script_package = "etcd-reconfiguration-restart.tar"
src_local = "parcel-center/{0}".format(bad_node_config["nodename"])
dst_remote = common.get_user_dir(bad_node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package, bad_node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}.tar && sudo ./{0}/{1}.sh".format("etcd-reconfiguration-restart", "restart-etcd-server")
if common.ssh_shell_with_password_input_paramiko(bad_node_config, commandline) == False:
sys.exit(1)
self.logger.info("Successfully restarting bad etcd server on node {0}".format(bad_node_config["nodename"]))
if self.clean_flag:
self.delete_packege(bad_node_config)
def job_executer_add_node_to_etcd_cluster(self):
com = self.cluster_object_model
self.logger.info("Find a alive etcd node in the cluster")
# Directly find the leader node.
good_node_config = pai_common.get_etcd_leader_node(com)
if good_node_config is None:
self.logger.error("Unable to find the etcd leader node.")
sys.exit(1)
self.logger.info(
"------------ package wrapper is working now ! -------------------- "
)
self.prepare_package(good_node_config, "add-master-node-task-one")
self.logger.info(
"------------ package wrapper's work finished ! ------------------- "
)
script_package = "add-master-node-task-one.tar"
src_local = "parcel-center/{0}".format(good_node_config["nodename"])
dst_remote = common.get_user_dir(good_node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
good_node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}".format(script_package)
if common.ssh_shell_with_password_input_paramiko(
good_node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully extract the script package for add-master-node-task-one!"
)
commandline = "sudo /bin/bash {0}/{1}.sh {2} {3}".format(
"add-master-node-task-one", "add-member-to-etcd-cluster",
self.node_config['hostip'], self.node_config['etcdid'])
if common.ssh_shell_with_password_input_paramiko(
good_node_config, commandline) == False:
sys.exit(1)
self.logger.info(
"Successfully add the new master into the etcd cluster.")
if self.clean_flag:
self.delete_packege(good_node_config)
self.remote_host_cleaner(good_node_config,
"add-master-node-task-one")
def job_executer(self):
print "repair job begins !"
# sftp your script to remote host with paramiko.
srcipt_package = "repair.tar"
src_local = "parcel-center/{0}".format(self.node_config["nodename"])
dst_remote = common.get_user_dir(self.node_config)
if common.sftp_paramiko(src_local, dst_remote, srcipt_package, self.node_config) == False:
return
commandline = "tar -xvf repair.tar && sudo ./repair/repair-worker-node.sh"
if common.ssh_shell_with_password_input_paramiko(self.node_config, commandline) == False:
return
print "Successfully running repair job on node {0}".format(self.node_config["nodename"])
def job_execute_stop_etcd_on_target_node(self):
self.logger.info("---- package wrapper is working now! ----")
self.prepare_package(self.node_config, "stop-etcd-on-target-node")
self.logger.info("---- package wrapper's work finished ----")
self.logger.info(
"Begin to execute the job : stop-etcd-on-target-node.")
self.logger.info("Stop the etcd server on host [{0}]".format(
self.node_config['nodename']))
script_package = "stop-etcd-on-target-node.tar"
src_local = "parcel-center/{0}".format(self.node_config["nodename"])
dst_remote = common.get_user_dir(self.node_config)
if common.sftp_paramiko(src_local, dst_remote, script_package,
self.node_config) == False:
sys.exit(1)
commandline = "tar -xvf {0}.tar && sudo /bin/bash {0}/stop-etcd-server.sh".format(
"stop-etcd-on-target-node")
if common.ssh_shell_with_password_input_paramiko(
self.node_config, commandline) == False:
sys.exit(1)
self.logger.info("Successfully stoping etcd server on node {0}".format(
self.node_config["nodename"]))
if self.clean_flag == True:
self.logger.info("---- package cleaner is working now! ----")
self.delete_packege(self.node_config)
self.logger.info("---- package cleaner's work finished! ----")
self.logger.info("---- remote host cleaner is working now! ----")
self.remote_host_cleaner(self.node_config,
"stop-etcd-on-target-node")
self.logger.info("---- remote host cleaning job finished! ")
