def start_abnormal_test_services():
try:
for host in config.etcd_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo rm -rf /etcd/default.etcd"
shell_operator.ssh_exec(ssh, ori_cmd)
etcd_cmd = "cd etcdrun && sudo nohup ./run.sh new &"
shell_operator.ssh_background_exec2(ssh, etcd_cmd)
ori_cmd = "ps -ef|grep -v grep | grep -w etcd | awk '{print $2}'"
time.sleep(2)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
logger.debug("etcd pid is %s"%rs[1])
assert rs[1] != [], "up etcd fail"
for host in config.mds_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
mds_cmd = "sudo nohup /usr/bin/curve-mds --confPath=/etc/curve/mds.conf &"
shell_operator.ssh_background_exec2(ssh, mds_cmd)
time.sleep(1)
ori_cmd = "ps -ef|grep -v grep | grep -v curve-mds.log | grep -v sudo | grep -w curve-mds | awk '{print $2}'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] != [], "up mds fail"
logger.debug("mds pid is %s"%rs[1])
for host in config.snap_server_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "cd snapshot/temp && sudo nohup curve-snapshotcloneserver -conf=/etc/curve/snapshot_clone_server.conf &"
shell_operator.ssh_background_exec2(ssh, ori_cmd)
except Exception:
logger.error("up servers fail.")
raise
def create_pool():
ssh = shell_operator.create_ssh_connect(config.mds_list[0], 1046,
config.abnormal_user)
mds = []
mds_addrs = ""
for mds_host in config.mds_list:
mds.append(mds_host + ":6666")
mds_addrs = ",".join(mds)
physical_pool = "curve-tool -cluster_map=topo.json -mds_addr=%s\
-op=create_physicalpool" % (mds_addrs)
rs = shell_operator.ssh_exec(ssh, physical_pool)
if rs[3] == 0:
logger.info("create physical pool sucess")
else:
assert False, "create physical fail ,msg is %s" % rs[2]
for host in config.chunkserver_list:
ssh2 = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo nohup ./chunkserver_ctl.sh start all &"
shell_operator.ssh_background_exec2(ssh2, ori_cmd)
time.sleep(60)
logical_pool = "curve-tool -cluster_map=topo.json -mds_addr=%s\
-op=create_logicalpool" % (mds_addrs)
rs = shell_operator.ssh_exec(ssh, logical_pool)
time.sleep(180)
def add_config_file():
for host in config.mds_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo cp -r /etc/curve-bak /etc/curve"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"add host %s config fail,error is %s"%(host,rs[2])
for host in config.chunkserver_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo cp -r /etc/curve-bak /etc/curve"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"add host %s config fail,error is %s"%(host,rs[2])
def install_deb():
try:
# mkdeb_url = config.curve_workspace + "mk-deb.sh"
# exec_mkdeb = "bash %s"%mkdeb_url
# shell_operator.run_exec2(exec_mkdeb)
cmd = "ls %scurve-mds*.deb" % config.curve_workspace
mds_deb = shell_operator.run_exec2(cmd)
version = mds_deb.split('+')[1]
for host in config.mds_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite *%s* aws-sdk_1.0_amd64.deb" % version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "mds install deb fail,error is %s %s" % (rs[1],
rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
for host in config.client_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite curve-sdk*%s*" % version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "sdk install deb fail,error is %s %s" % (rs[1],
rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
for host in config.chunkserver_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/" %\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite curve-chunkserver*%s* curve-tools*%s* aws-sdk_1.0_amd64.deb" % (
version, version)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[
3] == 0, "chunkserver install deb fail,error is %s %s" % (
rs[1], rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
except Exception:
logger.error("install deb fail.")
raise
def get_test_dir_file_md5():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
test_dir = os.path.join(config.fs_mount_path,config.fs_mount_dir[1])
ori_cmd ="cd " + test_dir + " && find vdb.1_1.dir/ -name 'vdb_*' -type f -print0 | xargs -0 md5sum > md5_1"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"get file md5_1 error, output %s"%rs[1]
def check_corefile():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo ls /corefile |grep core"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] != []:
assert False,"/corefile have coredump file,is %s"%rs[1]
def clean_fs_kernel_log():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo logrotate -vf /etc/logrotate.d/rsyslog"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0," rollback log fail, %s"%rs[1]
ssh.close()
def start_fs_fio():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo supervisorctl stop all && sudo supervisorctl reload"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
time.sleep(5)
ssh.close()
def check_vdbench_output():
try:
ssh = shell_operator.create_ssh_connect(config.fs_test_client[0], 1046,
config.abnormal_user)
ori_cmd = "grep \"Vdbench execution completed successfully\" /home/nbs/tools/vdbench/output -R"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] == []:
t = time.time()
ori_cmd = "mv /home/nbs/tools/vdbench/output /home/nbs/vdbench-output/output-%d" % int(
t)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ori_cmd = "cat /home/nbs/tools/vdbench/output/errorlog.html | grep 'Corrupted data block'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] != []:
error_mes = rs[1][0].strip()
error_file = re.search(r'(?<=file=).*(?=;)', error_mes).group()
ori_cmd = "sudo stat %s > /home/nbs/vdbench-output/output-%d/inode" % (
error_file, int(t))
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert False, "vdbench test error,save log to test client /home/nbs/vdbench-output/output-%d" % int(
t)
except Exception as e:
ssh.close()
raise
ssh.close()
def init_nbd_vol(check_md5=True, lazy="True"):
ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046,
config.abnormal_user)
try:
name = "volume-snapshot" + "-" + str(check_md5) + "-" + lazy
thrash = NbdThrash(ssh, name)
vol_size = 10 #GB
thrash.nbd_create(vol_size)
# nbd_dev = "nbd3"
thrash.nbd_map()
time.sleep(5)
thrash.nbd_getdev()
if check_md5 == True:
init_data = vol_size * 1024
thrash.write_data_full(init_data)
else:
init_data = 5000 #MB
thrash.write_data(init_data)
time.sleep(60)
config.snapshot_volid = name
config.snapshot_thrash = thrash
thrash.check_md5 = check_md5
except:
logger2.error("create snapshot nbd fail")
raise
def test_fs_process_loss_package(process_name, percent):
if process_name == "mds":
process_list = list(config.fs_mds)
elif process_name == "metaserver":
process_list = list(config.fs_metaserver)
elif process_name == "etcd":
process_list = list(config.fs_etcd)
elif process_name == "fuseclient":
process_list = list(config.fs_test_client)
test_host = random.choice(process_list)
ssh = shell_operator.create_ssh_connect(test_host, 1046,
config.abnormal_user)
dev = fault_inject.get_hostip_dev(ssh, test_host)
logger.info("|------begin test host %s dev %s loss package------|" %
(test_host, dev))
try:
fault_inject.package_loss_all(ssh, dev, percent)
fault_inject.show_tc_inject(ssh, dev)
# check_nbd_iops(1)
except Exception as e:
raise
finally:
time.sleep(60)
fault_inject.cancel_tc_inject(ssh, dev)
def test_out_metaserver_copyset():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
ori_cmd = "/home/nbs/.curveadm/bin/curveadm status --role=metaserver |grep metaserver| awk '{print $1}'"
rs = shell_operator.run_exec2(ori_cmd)
logger.info("rs is %s" % rs)
rs = rs.split('\n')
meta_docker = random.choice(rs)
ori_cmd = "/home/nbs/.curveadm/bin/curveadm stop --id=%s" % meta_docker
logger.info("|------begin test out one metaserver,docker %s------|" %
(meta_docker))
try:
rs = shell_operator.run_exec(ori_cmd)
assert rs == 0, "stop metaserver fail"
time.sleep(120)
# begin_num = get_metaserver_copyset_num(chunkserver_host,cs_list[0])
starttime = time.time()
while time.time() - starttime < 1200:
status = check_fs_copyset_status()
if status == True:
break
time.sleep(60)
logger.info("copyset is unhealthy")
assert status == True, "cluster metaserver %s not recover finish in %d" % (
meta_docker, 120)
except Exception as e:
# raise AssertionError()
logger.error("error is %s" % e)
test_start_process("metaserver")
raise
def start_nebd():
cmd = "ls nebd/nebd*.deb"
nebd_deb = shell_operator.run_exec2(cmd)
version = nebd_deb.split('+')[1]
assert nebd_deb != "","can not get nebd deb"
for host in config.client_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %snebd/*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite nebd_*%s"%version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"install nebd deb fail,error is %s"%rs
rm_deb = "rm nebd_*%s"%version
shell_operator.ssh_exec(ssh, rm_deb)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 nebd/etc/nebd/*.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = "sudo cp nebd-client.conf nebd-server.conf /etc/nebd/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"cp %s nebd conf fail"%host
ori_cmd = "sudo nebd-daemon start"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[3] != 0:
logger.debug("nebd start fail,error is %s"%rs[1])
ori_cmd == "sudo nebd-daemon restart"
rs2 = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs2[3] == 0,"restart nebd fail, return is %s"%rs2[1]
time.sleep(5)
ori_cmd = "ps -ef|grep nebd-server | grep -v daemon |grep -v grep |awk '{print $2}'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] != "","start nebd fail!"
def clean_corefile():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
ori_cmd = "sudo mv /corefile/core* /corefile/backup/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ssh.close()
def check_fuse_iops(limit=1):
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
port_list = []
for mnt in config.fs_mount_dir:
ori_cmd = "ps -ef|grep failover/%s | grep curve-fuse | grep -v grep | awk '{print $2}'" % mnt
rs = shell_operator.ssh_exec(ssh, ori_cmd)
pid = "".join(rs[1]).strip()
ori_cmd = "sudo netstat -lntp |grep %s |awk '{print $4}'" % pid
rs = shell_operator.ssh_exec(ssh, ori_cmd)
port_list.append("".join(rs[1]).strip())
for port in port_list:
# port = port.strip()
logger.info("get port %s ops" % port)
ori_cmd = "sudo curl -s http://" + port + "/vars" + " | grep \'user_write_bps :\'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "get bps fail,rs is %s" % rs[1]
write_bps = "".join(rs[1]).strip().split(":")[-1]
logger.info("now port %s bps is %s" % (port, write_bps))
if write_bps.isdigit():
assert int(
write_bps
) > limit, "get port %s user_write_bps %s is lower than %d" % (
port, write_bps, limit)
def test_recover_snapshot(lazy="true"):
logger2.info("------------begin test recover snapshot----------")
ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user)
vol_id = config.snapshot_volid
snapshot_uuid = create_vol_snapshot(vol_id)
starttime = time.time()
final = False
time.sleep(5)
while time.time() - starttime < config.snapshot_timeout:
rc = get_snapshot_status(vol_id,snapshot_uuid)
if rc["Progress"] == 100 and rc["Status"] == 0 :
final = True
break
else:
time.sleep(60)
if final == True:
try:
first_md5 = get_vol_md5(vol_id)
config.snapshot_thrash.write_data(8000)
config.snapshot_thrash.nbd_unmap()
recover_task = recover_snapshot(vol_id,snapshot_uuid,lazy)
final_recover = False
time.sleep(5)
starttime = time.time()
status = 0
if lazy == "true":
second_md5 = get_vol_md5(vol_id)
assert first_md5 == second_md5,"vol md5 not same after lazy recover,fisrt is %s,recovered is %s"(first_md5,second_md5)
flatten_clone_vol(recover_task)
starttime = time.time()
final = False
while time.time() - starttime < config.snapshot_timeout:
rc = get_clone_status(recover_task)
if rc["TaskStatus"] == 0 and rc["TaskType"] == 1:
final_recover = True
break
else:
time.sleep(60)
else:
final_recover = False
starttime = time.time()
while time.time() - starttime < config.snapshot_timeout:
rc = get_clone_status(recover_task)
if rc["TaskStatus"] == 0 and rc["TaskType"] == 1:
final_recover = True
break
else:
time.sleep(60)
if final_recover == True:
second_md5 = get_vol_md5(vol_id)
else:
assert False,"recover vol %s fail in %d s"%(vol_id,config.snapshot_timeout)
if config.snapshot_thrash.check_md5 == True:
assert first_md5 == second_md5,"vol md5 not same after recover,fisrt is %s,recovered is %s"(first_md5,second_md5)
config.snapshot_thrash.nbd_map()
config.snapshot_thrash.nbd_getdev()
except:
raise
delete_vol_snapshot(vol_id,snapshot_uuid)
check_snapshot_delete(vol_id,snapshot_uuid)
def check_fuse_mount_success(fs_mount_dir=config.fs_mount_dir):
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
for mount_dir in fs_mount_dir:
grep_cmd = "ps -ef | grep curve-fuse | grep failover/%s | grep -v grep | awk '{print $2}' " % mount_dir
rs = shell_operator.ssh_exec(ssh, grep_cmd)
pid = ""
if rs[1] != []:
pid = rs[1][0].strip()
logger.info("pid=%s" % pid)
if pid:
logger.debug("process is: %s" % pid)
try:
ori_cmd = "stat -c %a " + "%s/%s" % (config.fs_mount_path,
mount_dir)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "stat dir %s fail,error is %s" % (
mount_dir, rs[1])
permission = "".join(rs[1]).strip()
assert permission == "1777", "dir mount fail,permission is %s" % (
mount_dir, permission)
except Exception:
logger.error(" mount test dir fail.")
raise
else:
assert False, "process %s not exsits" % mount_dir
ssh.close()
def nbd_all(name):
thrash_time = 0
ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046,
config.abnormal_user)
thrash = NbdThrash(ssh, name)
vol_size = random.choice(size_list)
thrash.nbd_create(vol_size)
while config.thrash_map == True:
try:
thrash.nbd_map()
time.sleep(5)
thrash.nbd_getdev()
thrash.nbd_unmap()
time.sleep(15)
thrash_time = thrash_time + 1
logger3.info("thrash_map_unmap time is %d,filename is %s" %
(thrash_time, name))
except:
thrash.nbd_delete()
logger3.error("map/unmap nbd %s fail" % name)
config.thrash_map = False
raise
else:
thrash.nbd_delete()
return thrash_time
def get_fuse_pid(mount_dir):
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
grep_cmd = "ps -ef | grep curve-fuse | grep %s | grep -v grep | awk '{print $2}' " % mount_dir
rs = shell_operator.ssh_exec(ssh,grep_cmd)
pid = "".join(rs[1]).strip()
logger.info("pid=%s" %pid)
return pid
def random_kill_snapshot():
snap_server = random.choice(config.snap_server_list)
logger2.info("begin to kill snapshotserver %s"%(snap_server))
kill_snapshotclone_server(snap_server)
time.sleep(10)
ssh = shell_operator.create_ssh_connect(snap_server, 1046, config.abnormal_user)
ori_cmd = "cd snapshot/temp && sudo nohup curve-snapshotcloneserver -conf=/etc/curve/snapshot_clone_server.conf &"
shell_operator.ssh_background_exec2(ssh, ori_cmd)
def stop_nebd():
for host in config.client_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "ps -ef|grep -v grep | grep nebd | awk '{print $2}' | sudo xargs kill -9"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[3] != 0:
logger.debug("snapshotcloneserver not up")
continue
def start_fs_vdbench():
test_client = config.fs_test_client[0]
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 robot/Resources/config/profile \
%s:~/tools/vdbench/"%(config.pravie_key_path,test_client)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "cd /home/nbs/tools/vdbench && sudo nohup ./vdbench -jn -f profile &"
rs = shell_operator.ssh_background_exec2(ssh, ori_cmd)
time.sleep(5)
ssh.close()
def test_fuse_client_cpu_stress(stress=80):
# client_host = random.choice(config.client_list)
client_host = config.fs_test_client[0]
logger.info("|------begin test fuse client cpu stress,host %s------|"%(client_host))
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 robot/Resources/keywords/cpu_stress.py \
%s:~/"%(config.pravie_key_path,client_host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
fault_inject.inject_cpu_stress(ssh,stress)
return ssh
def check_fs_io_error():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo grep \'error\' /var/log/kern.log -R | grep -v libpthread"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] != []:
ori_cmd = "sudo logrotate -vf /etc/logrotate.d/rsyslog"
shell_operator.ssh_exec(ssh, ori_cmd)
assert False," rwio error,log is %s"%rs[1]
ssh.close()
def test_fuse_client_mem_stress(stress=80):
client_host = config.fs_test_client[0]
logger.info("|------begin test fuse mem stress,host %s------|"%(client_host))
cmd = "free -g |grep Mem|awk \'{print $2}\'"
ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
rs = shell_operator.ssh_exec(ssh, cmd)
all_mem = int("".join(rs[1]).strip())
stress = all_mem * stress / 100
fault_inject.inject_mem_stress(ssh,stress)
return ssh
def mv_data():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
operator = "mv"
for name in config.fs_mount_dir:
test_dir = os.path.join(config.fs_mount_path,name)
ori_cmd = operator + ' ' + test_dir + '/' + name + ".0.0" + ' ' + test_dir + '/' + name + ".0.0.bak"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv fail"
ssh.close()
def cp_check_data():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
test_dir = config.fs_mount_path
operator = "cp"
for name in config.fs_mount_dir:
ori_cmd = operator + ' ' + test_dir + name + ".0.0" + ' ' + test_dir + name
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"cp fail"
ssh.close()
def attach_snapshot_vol(vol_uuid):
ori_cmd = "source OPENRC && nova list |grep %s | awk '{print $2}'"%config.vm_host
ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user)
rs = shell_operator.ssh_exec(ssh,ori_cmd,logger2)
vm_uuid = "".join(rs[1]).strip()
ori_cmd = "source OPENRC &&nova volume-attach %s %s"%((vm_uuid,vol_uuid))
rs = shell_operator.ssh_exec(ssh,ori_cmd,logger2)
logger2.info("exec cmd %s" % ori_cmd)
assert rs[3] == 0,"detach vol fail,return is %s"%rs[2]
logger2.info("exec cmd %s"%ori_cmd)
ssh.close()
def restart_cinder_server():
for client_host in config.client_list:
ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
ori_cmd = "sudo cp /usr/curvefs/curvefs.py /srv/stack/cinder/lib/python2.7/site-packages/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ori_cmd = "sudo cp /usr/curvefs/_curvefs.so /srv/stack/cinder/lib/python2.7/site-packages/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
time.sleep(2)
ori_cmd = "sudo service cinder-volume restart"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] == [],"rs is %s"%rs
def clean_env():
host_list = config.client_list + config.mds_list + config.chunkserver_list
host_list = list(set(host_list))
for host in host_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd1 = "sudo tc qdisc del dev bond0.106 root"
shell_operator.ssh_exec(ssh, ori_cmd1)
ori_cmd2 = "ps -ef|grep -v grep | grep memtester | awk '{print $2}'| sudo xargs kill -9"
shell_operator.ssh_exec(ssh, ori_cmd2)
ori_cmd3 = "ps -ef|grep -v grep | grep cpu_stress.py | awk '{print $2}'| sudo xargs kill -9"
shell_operator.ssh_exec(ssh, ori_cmd3)
