def __ssh_daemon_is_running(ip,user,pwd,parser): """ use ssh log in to the shell , check ssh is ready :param ip : FTOS ip :param user : FTOS user :param pwd : FTOS user password :param parser : parser: is a dict, get from Test config file """ t_start = time.time() while((time.time() - t_start) < float(parser["pre_wait_ssh_ready_time"])): time.sleep(float(1)) try: ssh = shell_server.get_ssh(ip , user , pwd) #獲得ssh except Exception: print " checking %s ssh " % user continue ssh.close() return True try: ssh = shell_server.get_ssh(ip , user , pwd) #獲得ssh except Exception: print " %s ssh not ready" % user return False; ssh.close()
def is_node_exists(cluster_name, node_name, parser): """ check is node in HAagent :param cluster_name : cluster name :param node_name : node name :param parser : is a dict get from base.configure """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) # get ssh object cmd = cmd_HAagent.overview_cmd() s_stdin, s_stdout, s_stderr = ssh.exec_command(cmd) overview = s_stdout.read() # get overview in host terminal ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) cluster_file_content = file.get_remote_file_content( parser["cluster_file_path"], ssh) # get cluster file content in nfs print overview print cluster_file_content ssh.close() if node_name in overview and cluster_file_content: return True return False
def preprocess_hostOS_vm_running(parser): """ preprocess vm become running :called func: preprocess_hostOS_vm :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh if FTVM.is_running(parser["vm_name"], parser["PrimaryOS_ip"], ssh): print 59 if "pre_hostOS_VM_restart" in parser.keys( ) and parser["pre_hostOS_VM_restart"] == "yes": #根據參數若VM需重新啟動 print 54 prepocess_hostOS_vm_restart(parser) print 55 time.sleep(float(parser["pre_hostOS_VM_boot_time"])) elif FTVM.is_shutoff(parser["vm_name"], parser["PrimaryOS_ip"], ssh): print 56 prepocess_hostOS_vm_start(parser) print 57 time.sleep(float(parser["pre_hostOS_VM_boot_time"])) print 58 if not FTVM.is_running(parser["vm_name"], parser["PrimaryOS_ip"], ssh): ssh.close() raise TA_error.Preprocess_Error("PrimaryOS VM: %s can not start" % parser["vm_name"]) ssh.close()
def preprocess_backupOS_FTsystem(parser): """ preprocess backupOS FTsystem check FTsystem status start/stop FTsystem raise exception if FTsystem can not start/stop """ if parser["pre_check_backupOS_FTsystem"] == "yes": ssh = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) status = FTsystem.get_status(ssh) if status == "not running" and parser["pre_backupOS_FTsystem_start"] == "yes": FTsystem.start(ssh) time.sleep(float(parser["pre_backupOS_FTsystem_start_time"])) if FTsystem.get_status(ssh) == "not running": ssh.close() raise TA_error.Preprocess_Error("backupOS FTsystem can not start") if status == "running" and parser["pre_backupOS_FTsystem_start"] == "no": FTsystem.stop(ssh) time.sleep(float(parser["pre_backupOS_FTsystem_start_time"])) if FTsystem.get_status(ssh) == "running": ssh.close() raise TA_error.Preprocess_Error("backupOS FTsystem can not stop") ssh.close()
def vm_running_in_slaveOS(parser): """ vm is running in SlaveOS or not :param parser: config :return: True/raise exception """ ssh = shell_server.get_ssh(parser["SlaveOS_ip"], parser["SlaveOS_usr"], parser["SlaveOS_pwd"]) #獲得ssh t_end = time.time() if "ast_vm_running_wait_time" in parser.keys( ): #若參數ast_vm_running_wait_time存在於parser,則進入 t_end = time.time() + float( parser["ast_vm_running_wait_time"]) #計算出等待之時間,並存於t_end while time.time() < t_end: #超過t_end則跳出迴圈 #每sleep一秒就詢問一次狀態 time.sleep(1) if FTVM.is_running(parser["vm_name"], parser["SlaveOS_ip"], ssh): #狀態為running就跳出迴圈 break if FTVM.is_running(parser["vm_name"], parser["SlaveOS_ip"], ssh): #若回傳之狀態是running,則test oracle通過,否則raise exception ssh.close() return True ssh.close() raise TA_error.Assert_Error("VM (name : %s) is not running in SlaveOS" % parser["vm_name"])
def libvirt_stop_start_ftvm(parser): """ detect when libvirt daemon shutdown , can HAAgent start ftvm :param parser: config :return: True/raise exception """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh cmd = cmd_HAagent.start_ftvm_cmd(parser["PrimaryOS_name"], parser["vm_name"]) s_stdin, s_stdout, s_stderr = ssh.exec_command("sudo " + cmd) out = s_stdout.read() expected = HAagent_terminal.Startvm_create_vm_failed % ( HAagent_terminal.Libvirt_connection_failed % HAagent_terminal.Libvirt_self_connection) + "\n" success = (out == expected) print out print expected FTsystem.start(ssh) #print success if success: return True raise TA_error.Assert_Error("libvirt stop then start ftvm fail")
def postprocess_slaveOS_vm_shutdown(parser): """ postprocess backupOS vm become shutdown :called func: postprocess_backupOS_vm :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["SlaveOS_ip"], parser["SlaveOS_usr"], parser["SlaveOS_pwd"]) #獲得ssh if FTVM.is_running(parser["vm_name"], parser["SlaveOS_ip"], ssh): FTVM.destroy(parser["vm_name"], parser["SlaveOS_ip"], ssh) time.sleep(float(parser["pos_slaveOS_VM_shutdown_time"])) elif FTVM.is_paused(parser["vm_name"], parser["SlaveOS_ip"], ssh): FTVM.resume(parser["vm_name"], parser["SlaveOS_ip"], ssh) FTVM.destroy(parser["vm_name"], parser["SlaveOS_ip"], ssh) time.sleep(float(parser["pos_slaveOS_VM_shutdown_time"])) times = 0 while times < 30: print "check slave os vm status" if FTVM.is_running(parser["vm_name"], parser["SlaveOS_ip"], ssh): print "destroy slave os vm " FTVM.destroy(parser["vm_name"], parser["SlaveOS_ip"], ssh) break time.sleep(float(1)) times += 1 if not FTVM.is_shutoff(parser["vm_name"], parser["SlaveOS_ip"], ssh): ssh.close() raise TA_error.Postprocess_Error("SlaveOS %s can not shutdown" % parser["vm_name"]) ssh.close()
def preprocess_slaveOS_FTsystem(parser): """ preprocess backupOS FTsystem part check FTsystem status start/stop FTsystem raise exception if FTsystem can not start/stop :called func: preprocess_backupOS_OS :param parser: is a dict, get from Test config file """ if parser["pre_check_slaveOS_FTsystem"] == "yes": ssh = shell_server.get_ssh(parser["SlaveOS_ip"], parser["SlaveOS_usr"], parser["SlaveOS_pwd"]) #獲得ssh status = FTsystem.get_status(ssh) if status == "not running" and parser[ "pre_slaveOS_FTsystem_start"] == "yes": #若狀態不為running且根據參數必需是running則進入 FTsystem.start(ssh) #透過ssh開啟libvirt time.sleep(float(parser["pre_slaveOS_FTsystem_start_time"])) if FTsystem.get_status( ssh) == "not running": #若狀態不為running則raise exception ssh.close() raise TA_error.Preprocess_Error("slaveOS FTsystem can not start") if status == "running" and parser[ "pre_slaveOS_FTsystem_start"] == "no": #若狀態為running且根據參數必需不是running則進入 FTsystem.stop(ssh) time.sleep(float(parser["pre_slaveOS_FTsystem_start_time"])) if FTsystem.get_status( ssh) == "running": #若狀態為running則raise exception ssh.close() raise TA_error.Preprocess_Error( "slaveOS FTsystem can not stop") ssh.close()
def vm_is_login_in_slaveOS(parser): """ vm is login in hostOS or not :param parser: config :return: True/raise exception """ if FTVM.is_login(parser["vm_name"], parser["TA_ip"], parser["TA_msg_sock_port"], int(parser["ast_vm_login_wait_time"] )): #若回傳VM登入完成,則test oracle通過,否則raise exception return True else: ssh = shell_server.get_ssh(parser["GuestOS_ip_S"], parser["GuestOS_usr"], parser["GuestOS_pwd"]) #獲得ssh cmd = "sudo %s" % parser["login_reply_path"] #print cmd s_stdin, s_stdout, s_stderr = ssh.exec_command(cmd) #透過ssh執行指令 if FTVM.is_login(parser["vm_name"], parser["TA_ip"], parser["TA_msg_sock_port"], int(parser["ast_vm_login_wait_time"] )): #若回傳VM登入完成,則test oracle通過,否則raise exception return True ssh.close() raise TA_error.Assert_Error("VM (name : %s) is not login in backupOS" % parser["vm_name"])
def preprocess_slaveOS_HAagent(parser): ssh = shell_server.get_ssh(parser["SlaveOS_ip"], parser["SlaveOS_usr"], parser["SlaveOS_pwd"]) #獲得ssh if HAagent.is_running(ssh, parser): ssh.close() return True ssh.close() raise TA_error.Preprocess_Error("Slave OS HAAgent process not ready")
def __reset_pid(node , parser): if node == "primary": ssh = shell_server.get_ssh(parser["PrimaryOS_ip"] , parser["PrimaryOS_usr"] , parser["PrimaryOS_pwd"]) #獲得ssh ssh.exec_command("sudo rm /home/primary/Desktop/pid.txt") elif node == "backup": ssh = shell_server.get_ssh(parser["BackupOS_ip"] , parser["BackupOS_usr"] , parser["BackupOS_pwd"]) #獲得ssh ssh.exec_command("sudo rm /home/backup-node/Desktop/pid.txt") elif node == "slave": ssh = shell_server.get_ssh(parser["SlaveOS_ip"] , parser["SlaveOS_usr"] , parser["SlaveOS_pwd"]) #獲得ssh ssh.exec_command("sudo rm /home/slave/Desktop/pid.txt") ssh.close()
def stop_libvirt_process(parser): ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh FTsystem.stop(ssh) if FTsystem.get_status(ssh) == "running": raise TA_error.Process_Error("libvirt in host OS cannot stop") ssh.close()
def postprocess_NFS_OS_reboot(parser): """ when test case done , Host OS reboot :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) #獲得ssh if FTOS.OS_is_running(parser["NFS_ip"], parser): FTOS.reboot(ssh)
def postprocess_slaveOS_HAAgent(parser): ssh = shell_server.get_ssh(parser["SlaveOS_ip"], parser["SlaveOS_usr"], parser["SlaveOS_pwd"]) #獲得ssh if HAagent.is_running(ssh, parser): HAagent.exit(parser, ssh) if HAagent.is_running(ssh, parser): ssh.close() raise TA_error.Postprocess_Error("Slave HAAgent cannot shutdown") else: pass
def hostOS_role_is_Slave_on_BackupOS(parser): ssh = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) #獲得ssh role = mmsh.inforole(parser["PrimaryOS_name"], ssh) ssh.close() if role == "slave": return True raise TA_error.Assert_Error("Host (name : %s) role is not slave" % parser["PrimaryOS_name"])
def exec_L1_hostOS_crasher(parser): """ execute level 1 crasher in hostOS """ ssh = shell_server.get_ssh(parser["HostOS_ip"] , parser["HostOS_usr"] , parser["HostOS_pwd"]) f_path = parser["HostOS_process_dir"]+"L1_crasher" s_stdin, s_stdout, s_stderr = ssh.exec_command("sudo "+cmd) ssh.close()
def detect_overview(parser): """ detect remove node or not :param parser: config :return: True/raise exception """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh out = HAagent.overview(parser, ssh)
def postprocess_NFS_reset(parser): """ when test done , clear nfs file :param parser: is a dict, get from Test config file :param ssh : shell server """ ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) #獲得ssh NFS.reset(parser, ssh) ssh.close()
def preprocess_NFS_OS(parser): if FTOS.OS_is_running(parser["NFS_ip"], parser) == False: #FTOS.L1_boot(parser["NFS_NetworkAdaptor"]) if FTOS.OS_is_running(parser["NFS_ip"], parser) == False: raise TA_error.Preprocess_Error("NFS node can not start") if FTOS.ssh_is_ready(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"], parser) == False: raise TA_error.Preprocess_Error("NFS node ssh can not access") ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) #獲得ssh
def exec_overview(parser): """ HAagent overview :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh #HAagent.quick_create_cluster(parser, ssh) time.sleep(1) ssh.close()
def exec_non_primary_de_node(parser): """ use non-primary node remove node :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh #use primary node create cluster HAagent.create_cluster(parser["Cluster_name"], parser["PrimaryOS_name"], parser["PrimaryOS_ipmb"], parser["Shelf_ip"], parser, ssh) HAagent.add_backup_node(parser, ssh) time.sleep(float(parser["pro_wait_add_node_time"])) #use backup node to remove node backup = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) #獲得ssh HAagent.rm_node(parser["Cluster_name"], parser["PrimaryOS_name"], parser, backup) backup.close()
def exec_L1_vm_crasher(parser): """ execute level 1 crasher in vm """ ssh = shell_server.get_ssh(parser["GuestOS_ip"] , parser["GuestOS_usr"] , parser["GuestOS_pwd"]) cmd = data_dir.OS_PROCESS_DIR+"L1_crasher" print cmd s_stdin, s_stdout, s_stderr = ssh.exec_command("sudo "+cmd) ssh.close()
def kill_vm_process(parser): """ kill vm process on hostOS """ ssh = shell_server.get_ssh(parser["HostOS_ip"] , parser["HostOS_usr"] , parser["HostOS_pwd"]) pid = FTVM.get_pid(parser["vm_name"], parser["HostOS_ip"], ssh) cmd = cmd_kill.kill_cmd(pid, 9) s_stdin, s_stdout, s_stderr = ssh.exec_command("sudo "+cmd) ssh.close()
def kill_libvirt_process(parser): """ kill libvirt process on hostOS """ ssh = shell_server.get_ssh(parser["GuestOS_ip"] , parser["GuestOS_usr"] , parser["GuestOS_pwd"]) pid = FTsystem.get_pid(ssh) cmd = cmd_kill.kill_cmd(pid, 9) s_stdin, s_stdout, s_stderr = ssh.exec_command("sudo "+cmd) ssh.close()
def exec_add_duplicate_node(parser): """ HAagent add duplicate node to cluster :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh HAagent.create_cluster(parser["Cluster_name"], parser["PrimaryOS_name"], parser["PrimaryOS_ipmb"], parser["Shelf_ip"], parser, ssh) ssh.close()
def exec_non_primary_de_cluster(parser): """ HAagent add node to cluster :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh # create cluster HAagent.create_cluster(parser["Cluster_name"], parser["PrimaryOS_name"], parser["PrimaryOS_ipmb"], parser["Shelf_ip"], parser, ssh) HAagent.add_backup_node(parser, ssh) time.sleep(float(1)) ssh.close() #use backup node to decluster ssh = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) #獲得ssh HAagent.de_cluster(parser["Cluster_name"], parser, ssh) ssh.close()
def exec_primaryOS_network_isolation(parser): if "pro_wait_time_exe_L1_crasher" in parser.keys( ): #若pro_wait_time_exe_L1_crasher存在於parser time.sleep(int(parser["pro_wait_time_exe_L1_crasher"])) ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh cmd = "sudo ifconfig %s down" % (parser["PrimaryOS_network_interface"]) print cmd s_stdin, s_stdout, s_stderr = ssh.exec_command(cmd) #透過ssh執行指令 ssh.close() ssh = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) #獲得ssh #print "stdout",s_stdout.read() #print "stderr",s_stderr.read() ssh.close()
def FTsystem_running_in_hostOS(parser): """ FTsystem is running in hostOS or not return True/raise exception """ ssh = shell_server.get_ssh(parser["HostOS_ip"] , parser["HostOS_usr"] , parser["HostOS_pwd"]) if FTsystem.get_status(ssh) == "running": return True raise TA_error.Assert_Error("FTsystem is not running on hostOS")
def detect_primaryOS_crash(parser): ssh = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) #獲得ssh if mmsh.statehost( parser["PrimaryOS_name"], ssh ) == "shutdown": #若回傳之狀態是hardware shutdown,則test oracle通過,否則raise exception ssh.close() return True ssh.close() raise TA_error.Assert_Error( "Host (name : %s) has not detect host os crash " % parser["PrimaryOS_name"])
def detect_no_fail(parser): """ FTsystem find no fail or not :param parser: config :return: True/raise exception """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲取ssh if mmsh.infofail(parser["vm_name"], ssh) == "no fail": return True raise TA_error.Assert_Error("VM (name : %s) has fail in hostOS" % parser["vm_name"])
def get_node_role(name, parser): ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) cluster_file_content = file.get_remote_file_content( parser["cluster_file_path"], ssh) # get cluster file content in nfs try: res = json.loads(cluster_file_content)["nodes"][name]["role"] ssh.close() return role_parse(res) except KeyError: return "Key not found"
def detect_slaveOS_network_isolation_info(parser): ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) #獲取ssh fail = HAagent_info.get_node_infofail(parser["SlaveOS_name"], parser, ssh) expected = HAagent_terminal.Node_status_self_network_isolation success = (fail == expected) if success: return True raise TA_error.Assert_Error( "slave network isolation info fail , Fail msg : %s , Expected msg : %s" % fail, expected)
def exec_de_outer_cluster(parser): """ HAagent external cluster primary delete cluster :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh HAagent.create_cluster("test_c", parser["PrimaryOS_name"], parser["PrimaryOS_ipmb"], parser["Shelf_ip"], parser, ssh) HAagent.de_cluster("test_b", parser, ssh) ssh.close()
def recovery_vm_reboot(parser): """ FTsystem recover vm reboot or not :param parser: config :return: True/raise exception """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲取ssh if mmsh.inforecover(parser["vm_name"], ssh) == "vm reboot": return True raise TA_error.Assert_Error("VM (name : %s) has no recovery : vm reboot" % parser["vm_name"])
def exec_add_node(parser): """ HAagnet add node (primary , backup , slave) :param parser: is a dict, get from Test config file """ ssh = shell_server.get_ssh(parser["PrimaryOS_ip"], parser["PrimaryOS_usr"], parser["PrimaryOS_pwd"]) #獲得ssh backup = shell_server.get_ssh(parser["BackupOS_ip"], parser["BackupOS_usr"], parser["BackupOS_pwd"]) #獲得ssh HAagent.create_cluster(parser["Cluster_name"], parser["PrimaryOS_name"], parser["PrimaryOS_ipmb"], parser["Shelf_ip"], parser, ssh) time.sleep(float(parser["pro_wait_add_node_time"])) HAagent.add_backup_node(parser, ssh) time.sleep(float(parser["pro_wait_add_node_time"])) HAagent.add_slave_node(parser, ssh) time.sleep(float(parser["pro_wait_add_node_time"])) ssh.close()
def detect_backupOS_crash_info(parser): ssh = shell_server.get_ssh(parser["NFS_ip"], parser["NFS_usr"], parser["NFS_pwd"]) #獲取ssh fail = HAagent_info.get_node_infofail(parser["BackupOS_name"], parser, ssh) expected = HAagent_terminal.Node_status_hostdown success = (fail == expected) if success: return True raise TA_error.Assert_Error( "backup OS crash info fail , Fail msg : %s , Expected msg : %s" % fail, expected)
get vm process id in OS """ pid_file_path = data_dir.VM_PID_DIR+vm_name+".pid" cmd = "sudo cat %s" % pid_file_path s_stdin, s_stdout, s_stderr = ssh.exec_command(cmd) pid = s_stdout.read() if pid != "": return int(pid) return False if __name__ == '__main__': """ if get_vm_status("VM1", "140.115.53.42") != "running": start("VM1", "140.115.53.42") get_vm_status("VM1", "140.115.53.42") """ #print "start" #if get_vm_status("VM1", "140.115.53.42") == "running": #print "in" #get_vm_status("VM1", "140.115.53.42") #print is_shutoff("VM1", "140.115.53.42") #restart("VM1", "140.115.53.42") #shutdown("VM1", "140.115.53.42") ssh = shell_server.get_ssh("140.115.53.42", "ting", "oolabting") print get_pid("VM1", "140.115.53.42", ssh)