def start_abnormal_test_services(): try: for host in config.etcd_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo rm -rf /etcd/default.etcd" shell_operator.ssh_exec(ssh, ori_cmd) etcd_cmd = "cd etcdrun && sudo nohup ./run.sh new &" shell_operator.ssh_background_exec2(ssh, etcd_cmd) ori_cmd = "ps -ef|grep -v grep | grep -w etcd | awk '{print $2}'" time.sleep(2) rs = shell_operator.ssh_exec(ssh, ori_cmd) logger.debug("etcd pid is %s"%rs[1]) assert rs[1] != [], "up etcd fail" for host in config.mds_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) mds_cmd = "sudo nohup /usr/bin/curve-mds --confPath=/etc/curve/mds.conf &" shell_operator.ssh_background_exec2(ssh, mds_cmd) time.sleep(1) ori_cmd = "ps -ef|grep -v grep | grep -v curve-mds.log | grep -v sudo | grep -w curve-mds | awk '{print $2}'" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[1] != [], "up mds fail" logger.debug("mds pid is %s"%rs[1]) for host in config.snap_server_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "cd snapshot/temp && sudo nohup curve-snapshotcloneserver -conf=/etc/curve/snapshot_clone_server.conf &" shell_operator.ssh_background_exec2(ssh, ori_cmd) except Exception: logger.error("up servers fail.") raise
def create_pool(): ssh = shell_operator.create_ssh_connect(config.mds_list[0], 1046, config.abnormal_user) mds = [] mds_addrs = "" for mds_host in config.mds_list: mds.append(mds_host + ":6666") mds_addrs = ",".join(mds) physical_pool = "curve-tool -cluster_map=topo.json -mds_addr=%s\ -op=create_physicalpool" % (mds_addrs) rs = shell_operator.ssh_exec(ssh, physical_pool) if rs[3] == 0: logger.info("create physical pool sucess") else: assert False, "create physical fail ,msg is %s" % rs[2] for host in config.chunkserver_list: ssh2 = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo nohup ./chunkserver_ctl.sh start all &" shell_operator.ssh_background_exec2(ssh2, ori_cmd) time.sleep(60) logical_pool = "curve-tool -cluster_map=topo.json -mds_addr=%s\ -op=create_logicalpool" % (mds_addrs) rs = shell_operator.ssh_exec(ssh, logical_pool) time.sleep(180)
def add_config_file(): for host in config.mds_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo cp -r /etc/curve-bak /etc/curve" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"add host %s config fail,error is %s"%(host,rs[2]) for host in config.chunkserver_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo cp -r /etc/curve-bak /etc/curve" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"add host %s config fail,error is %s"%(host,rs[2])
def install_deb(): try: # mkdeb_url = config.curve_workspace + "mk-deb.sh" # exec_mkdeb = "bash %s"%mkdeb_url # shell_operator.run_exec2(exec_mkdeb) cmd = "ls %scurve-mds*.deb" % config.curve_workspace mds_deb = shell_operator.run_exec2(cmd) version = mds_deb.split('+')[1] for host in config.mds_list: cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\ (config.pravie_key_path,config.curve_workspace,host) shell_operator.run_exec2(cmd) ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo dpkg -i --force-overwrite *%s* aws-sdk_1.0_amd64.deb" % version rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0, "mds install deb fail,error is %s %s" % (rs[1], rs[2]) rm_deb = "rm *%s*" % version shell_operator.ssh_exec(ssh, rm_deb) for host in config.client_list: cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\ (config.pravie_key_path,config.curve_workspace,host) shell_operator.run_exec2(cmd) ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo dpkg -i --force-overwrite curve-sdk*%s*" % version rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0, "sdk install deb fail,error is %s %s" % (rs[1], rs[2]) rm_deb = "rm *%s*" % version shell_operator.ssh_exec(ssh, rm_deb) for host in config.chunkserver_list: cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/" %\ (config.pravie_key_path,config.curve_workspace,host) shell_operator.run_exec2(cmd) ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo dpkg -i --force-overwrite curve-chunkserver*%s* curve-tools*%s* aws-sdk_1.0_amd64.deb" % ( version, version) rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[ 3] == 0, "chunkserver install deb fail,error is %s %s" % ( rs[1], rs[2]) rm_deb = "rm *%s*" % version shell_operator.ssh_exec(ssh, rm_deb) except Exception: logger.error("install deb fail.") raise
def get_test_dir_file_md5(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) test_dir = os.path.join(config.fs_mount_path,config.fs_mount_dir[1]) ori_cmd ="cd " + test_dir + " && find vdb.1_1.dir/ -name 'vdb_*' -type f -print0 | xargs -0 md5sum > md5_1" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"get file md5_1 error, output %s"%rs[1]
def check_corefile(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "sudo ls /corefile |grep core" rs = shell_operator.ssh_exec(ssh, ori_cmd) if rs[1] != []: assert False,"/corefile have coredump file,is %s"%rs[1]
def clean_fs_kernel_log(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "sudo logrotate -vf /etc/logrotate.d/rsyslog" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0," rollback log fail, %s"%rs[1] ssh.close()
def start_fs_fio(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "sudo supervisorctl stop all && sudo supervisorctl reload" rs = shell_operator.ssh_exec(ssh, ori_cmd) time.sleep(5) ssh.close()
def check_vdbench_output(): try: ssh = shell_operator.create_ssh_connect(config.fs_test_client[0], 1046, config.abnormal_user) ori_cmd = "grep \"Vdbench execution completed successfully\" /home/nbs/tools/vdbench/output -R" rs = shell_operator.ssh_exec(ssh, ori_cmd) if rs[1] == []: t = time.time() ori_cmd = "mv /home/nbs/tools/vdbench/output /home/nbs/vdbench-output/output-%d" % int( t) rs = shell_operator.ssh_exec(ssh, ori_cmd) ori_cmd = "cat /home/nbs/tools/vdbench/output/errorlog.html | grep 'Corrupted data block'" rs = shell_operator.ssh_exec(ssh, ori_cmd) if rs[1] != []: error_mes = rs[1][0].strip() error_file = re.search(r'(?<=file=).*(?=;)', error_mes).group() ori_cmd = "sudo stat %s > /home/nbs/vdbench-output/output-%d/inode" % ( error_file, int(t)) rs = shell_operator.ssh_exec(ssh, ori_cmd) assert False, "vdbench test error,save log to test client /home/nbs/vdbench-output/output-%d" % int( t) except Exception as e: ssh.close() raise ssh.close()
def init_nbd_vol(check_md5=True, lazy="True"): ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user) try: name = "volume-snapshot" + "-" + str(check_md5) + "-" + lazy thrash = NbdThrash(ssh, name) vol_size = 10 #GB thrash.nbd_create(vol_size) # nbd_dev = "nbd3" thrash.nbd_map() time.sleep(5) thrash.nbd_getdev() if check_md5 == True: init_data = vol_size * 1024 thrash.write_data_full(init_data) else: init_data = 5000 #MB thrash.write_data(init_data) time.sleep(60) config.snapshot_volid = name config.snapshot_thrash = thrash thrash.check_md5 = check_md5 except: logger2.error("create snapshot nbd fail") raise
def test_fs_process_loss_package(process_name, percent): if process_name == "mds": process_list = list(config.fs_mds) elif process_name == "metaserver": process_list = list(config.fs_metaserver) elif process_name == "etcd": process_list = list(config.fs_etcd) elif process_name == "fuseclient": process_list = list(config.fs_test_client) test_host = random.choice(process_list) ssh = shell_operator.create_ssh_connect(test_host, 1046, config.abnormal_user) dev = fault_inject.get_hostip_dev(ssh, test_host) logger.info("|------begin test host %s dev %s loss package------|" % (test_host, dev)) try: fault_inject.package_loss_all(ssh, dev, percent) fault_inject.show_tc_inject(ssh, dev) # check_nbd_iops(1) except Exception as e: raise finally: time.sleep(60) fault_inject.cancel_tc_inject(ssh, dev)
def test_out_metaserver_copyset(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "/home/nbs/.curveadm/bin/curveadm status --role=metaserver |grep metaserver| awk '{print $1}'" rs = shell_operator.run_exec2(ori_cmd) logger.info("rs is %s" % rs) rs = rs.split('\n') meta_docker = random.choice(rs) ori_cmd = "/home/nbs/.curveadm/bin/curveadm stop --id=%s" % meta_docker logger.info("|------begin test out one metaserver,docker %s------|" % (meta_docker)) try: rs = shell_operator.run_exec(ori_cmd) assert rs == 0, "stop metaserver fail" time.sleep(120) # begin_num = get_metaserver_copyset_num(chunkserver_host,cs_list[0]) starttime = time.time() while time.time() - starttime < 1200: status = check_fs_copyset_status() if status == True: break time.sleep(60) logger.info("copyset is unhealthy") assert status == True, "cluster metaserver %s not recover finish in %d" % ( meta_docker, 120) except Exception as e: # raise AssertionError() logger.error("error is %s" % e) test_start_process("metaserver") raise
def start_nebd(): cmd = "ls nebd/nebd*.deb" nebd_deb = shell_operator.run_exec2(cmd) version = nebd_deb.split('+')[1] assert nebd_deb != "","can not get nebd deb" for host in config.client_list: cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %snebd/*.deb %s:~/"%\ (config.pravie_key_path,config.curve_workspace,host) shell_operator.run_exec2(cmd) ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "sudo dpkg -i --force-overwrite nebd_*%s"%version rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"install nebd deb fail,error is %s"%rs rm_deb = "rm nebd_*%s"%version shell_operator.ssh_exec(ssh, rm_deb) cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 nebd/etc/nebd/*.conf %s:~/"%\ (config.pravie_key_path,host) shell_operator.run_exec2(cmd) ori_cmd = "sudo cp nebd-client.conf nebd-server.conf /etc/nebd/" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"cp %s nebd conf fail"%host ori_cmd = "sudo nebd-daemon start" rs = shell_operator.ssh_exec(ssh, ori_cmd) if rs[3] != 0: logger.debug("nebd start fail,error is %s"%rs[1]) ori_cmd == "sudo nebd-daemon restart" rs2 = shell_operator.ssh_exec(ssh, ori_cmd) assert rs2[3] == 0,"restart nebd fail, return is %s"%rs2[1] time.sleep(5) ori_cmd = "ps -ef|grep nebd-server | grep -v daemon |grep -v grep |awk '{print $2}'" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[1] != "","start nebd fail!"
def clean_corefile(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "sudo mv /corefile/core* /corefile/backup/" rs = shell_operator.ssh_exec(ssh, ori_cmd) ssh.close()
def check_fuse_iops(limit=1): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) port_list = [] for mnt in config.fs_mount_dir: ori_cmd = "ps -ef|grep failover/%s | grep curve-fuse | grep -v grep | awk '{print $2}'" % mnt rs = shell_operator.ssh_exec(ssh, ori_cmd) pid = "".join(rs[1]).strip() ori_cmd = "sudo netstat -lntp |grep %s |awk '{print $4}'" % pid rs = shell_operator.ssh_exec(ssh, ori_cmd) port_list.append("".join(rs[1]).strip()) for port in port_list: # port = port.strip() logger.info("get port %s ops" % port) ori_cmd = "sudo curl -s http://" + port + "/vars" + " | grep \'user_write_bps :\'" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0, "get bps fail,rs is %s" % rs[1] write_bps = "".join(rs[1]).strip().split(":")[-1] logger.info("now port %s bps is %s" % (port, write_bps)) if write_bps.isdigit(): assert int( write_bps ) > limit, "get port %s user_write_bps %s is lower than %d" % ( port, write_bps, limit)
def test_recover_snapshot(lazy="true"): logger2.info("------------begin test recover snapshot----------") ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user) vol_id = config.snapshot_volid snapshot_uuid = create_vol_snapshot(vol_id) starttime = time.time() final = False time.sleep(5) while time.time() - starttime < config.snapshot_timeout: rc = get_snapshot_status(vol_id,snapshot_uuid) if rc["Progress"] == 100 and rc["Status"] == 0 : final = True break else: time.sleep(60) if final == True: try: first_md5 = get_vol_md5(vol_id) config.snapshot_thrash.write_data(8000) config.snapshot_thrash.nbd_unmap() recover_task = recover_snapshot(vol_id,snapshot_uuid,lazy) final_recover = False time.sleep(5) starttime = time.time() status = 0 if lazy == "true": second_md5 = get_vol_md5(vol_id) assert first_md5 == second_md5,"vol md5 not same after lazy recover,fisrt is %s,recovered is %s"(first_md5,second_md5) flatten_clone_vol(recover_task) starttime = time.time() final = False while time.time() - starttime < config.snapshot_timeout: rc = get_clone_status(recover_task) if rc["TaskStatus"] == 0 and rc["TaskType"] == 1: final_recover = True break else: time.sleep(60) else: final_recover = False starttime = time.time() while time.time() - starttime < config.snapshot_timeout: rc = get_clone_status(recover_task) if rc["TaskStatus"] == 0 and rc["TaskType"] == 1: final_recover = True break else: time.sleep(60) if final_recover == True: second_md5 = get_vol_md5(vol_id) else: assert False,"recover vol %s fail in %d s"%(vol_id,config.snapshot_timeout) if config.snapshot_thrash.check_md5 == True: assert first_md5 == second_md5,"vol md5 not same after recover,fisrt is %s,recovered is %s"(first_md5,second_md5) config.snapshot_thrash.nbd_map() config.snapshot_thrash.nbd_getdev() except: raise delete_vol_snapshot(vol_id,snapshot_uuid) check_snapshot_delete(vol_id,snapshot_uuid)
def check_fuse_mount_success(fs_mount_dir=config.fs_mount_dir): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) for mount_dir in fs_mount_dir: grep_cmd = "ps -ef | grep curve-fuse | grep failover/%s | grep -v grep | awk '{print $2}' " % mount_dir rs = shell_operator.ssh_exec(ssh, grep_cmd) pid = "" if rs[1] != []: pid = rs[1][0].strip() logger.info("pid=%s" % pid) if pid: logger.debug("process is: %s" % pid) try: ori_cmd = "stat -c %a " + "%s/%s" % (config.fs_mount_path, mount_dir) rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0, "stat dir %s fail,error is %s" % ( mount_dir, rs[1]) permission = "".join(rs[1]).strip() assert permission == "1777", "dir mount fail,permission is %s" % ( mount_dir, permission) except Exception: logger.error(" mount test dir fail.") raise else: assert False, "process %s not exsits" % mount_dir ssh.close()
def nbd_all(name): thrash_time = 0 ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user) thrash = NbdThrash(ssh, name) vol_size = random.choice(size_list) thrash.nbd_create(vol_size) while config.thrash_map == True: try: thrash.nbd_map() time.sleep(5) thrash.nbd_getdev() thrash.nbd_unmap() time.sleep(15) thrash_time = thrash_time + 1 logger3.info("thrash_map_unmap time is %d,filename is %s" % (thrash_time, name)) except: thrash.nbd_delete() logger3.error("map/unmap nbd %s fail" % name) config.thrash_map = False raise else: thrash.nbd_delete() return thrash_time
def get_fuse_pid(mount_dir): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) grep_cmd = "ps -ef | grep curve-fuse | grep %s | grep -v grep | awk '{print $2}' " % mount_dir rs = shell_operator.ssh_exec(ssh,grep_cmd) pid = "".join(rs[1]).strip() logger.info("pid=%s" %pid) return pid
def random_kill_snapshot(): snap_server = random.choice(config.snap_server_list) logger2.info("begin to kill snapshotserver %s"%(snap_server)) kill_snapshotclone_server(snap_server) time.sleep(10) ssh = shell_operator.create_ssh_connect(snap_server, 1046, config.abnormal_user) ori_cmd = "cd snapshot/temp && sudo nohup curve-snapshotcloneserver -conf=/etc/curve/snapshot_clone_server.conf &" shell_operator.ssh_background_exec2(ssh, ori_cmd)
def stop_nebd(): for host in config.client_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd = "ps -ef|grep -v grep | grep nebd | awk '{print $2}' | sudo xargs kill -9" rs = shell_operator.ssh_exec(ssh, ori_cmd) if rs[3] != 0: logger.debug("snapshotcloneserver not up") continue
def start_fs_vdbench(): test_client = config.fs_test_client[0] cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 robot/Resources/config/profile \ %s:~/tools/vdbench/"%(config.pravie_key_path,test_client) shell_operator.run_exec2(cmd) ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "cd /home/nbs/tools/vdbench && sudo nohup ./vdbench -jn -f profile &" rs = shell_operator.ssh_background_exec2(ssh, ori_cmd) time.sleep(5) ssh.close()
def test_fuse_client_cpu_stress(stress=80): # client_host = random.choice(config.client_list) client_host = config.fs_test_client[0] logger.info("|------begin test fuse client cpu stress,host %s------|"%(client_host)) cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 robot/Resources/keywords/cpu_stress.py \ %s:~/"%(config.pravie_key_path,client_host) shell_operator.run_exec2(cmd) ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user) fault_inject.inject_cpu_stress(ssh,stress) return ssh
def check_fs_io_error(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) ori_cmd = "sudo grep \'error\' /var/log/kern.log -R | grep -v libpthread" rs = shell_operator.ssh_exec(ssh, ori_cmd) if rs[1] != []: ori_cmd = "sudo logrotate -vf /etc/logrotate.d/rsyslog" shell_operator.ssh_exec(ssh, ori_cmd) assert False," rwio error,log is %s"%rs[1] ssh.close()
def test_fuse_client_mem_stress(stress=80): client_host = config.fs_test_client[0] logger.info("|------begin test fuse mem stress,host %s------|"%(client_host)) cmd = "free -g |grep Mem|awk \'{print $2}\'" ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user) rs = shell_operator.ssh_exec(ssh, cmd) all_mem = int("".join(rs[1]).strip()) stress = all_mem * stress / 100 fault_inject.inject_mem_stress(ssh,stress) return ssh
def mv_data(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) operator = "mv" for name in config.fs_mount_dir: test_dir = os.path.join(config.fs_mount_path,name) ori_cmd = operator + ' ' + test_dir + '/' + name + ".0.0" + ' ' + test_dir + '/' + name + ".0.0.bak" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"mv fail" ssh.close()
def cp_check_data(): test_client = config.fs_test_client[0] ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user) test_dir = config.fs_mount_path operator = "cp" for name in config.fs_mount_dir: ori_cmd = operator + ' ' + test_dir + name + ".0.0" + ' ' + test_dir + name rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[3] == 0,"cp fail" ssh.close()
def attach_snapshot_vol(vol_uuid): ori_cmd = "source OPENRC && nova list |grep %s | awk '{print $2}'"%config.vm_host ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user) rs = shell_operator.ssh_exec(ssh,ori_cmd,logger2) vm_uuid = "".join(rs[1]).strip() ori_cmd = "source OPENRC &&nova volume-attach %s %s"%((vm_uuid,vol_uuid)) rs = shell_operator.ssh_exec(ssh,ori_cmd,logger2) logger2.info("exec cmd %s" % ori_cmd) assert rs[3] == 0,"detach vol fail,return is %s"%rs[2] logger2.info("exec cmd %s"%ori_cmd) ssh.close()
def restart_cinder_server(): for client_host in config.client_list: ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user) ori_cmd = "sudo cp /usr/curvefs/curvefs.py /srv/stack/cinder/lib/python2.7/site-packages/" rs = shell_operator.ssh_exec(ssh, ori_cmd) ori_cmd = "sudo cp /usr/curvefs/_curvefs.so /srv/stack/cinder/lib/python2.7/site-packages/" rs = shell_operator.ssh_exec(ssh, ori_cmd) time.sleep(2) ori_cmd = "sudo service cinder-volume restart" rs = shell_operator.ssh_exec(ssh, ori_cmd) assert rs[1] == [],"rs is %s"%rs
def clean_env(): host_list = config.client_list + config.mds_list + config.chunkserver_list host_list = list(set(host_list)) for host in host_list: ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user) ori_cmd1 = "sudo tc qdisc del dev bond0.106 root" shell_operator.ssh_exec(ssh, ori_cmd1) ori_cmd2 = "ps -ef|grep -v grep | grep memtester | awk '{print $2}'| sudo xargs kill -9" shell_operator.ssh_exec(ssh, ori_cmd2) ori_cmd3 = "ps -ef|grep -v grep | grep cpu_stress.py | awk '{print $2}'| sudo xargs kill -9" shell_operator.ssh_exec(ssh, ori_cmd3)