def create_pool():
ssh = shell_operator.create_ssh_connect(config.mds_list[0], 1046,
config.abnormal_user)
mds = []
mds_addrs = ""
for mds_host in config.mds_list:
mds.append(mds_host + ":6666")
mds_addrs = ",".join(mds)
physical_pool = "curve-tool -cluster_map=topo.json -mds_addr=%s\
-op=create_physicalpool" % (mds_addrs)
rs = shell_operator.ssh_exec(ssh, physical_pool)
if rs[3] == 0:
logger.info("create physical pool sucess")
else:
assert False, "create physical fail ,msg is %s" % rs[2]
for host in config.chunkserver_list:
ssh2 = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo nohup ./chunkserver_ctl.sh start all &"
shell_operator.ssh_background_exec2(ssh2, ori_cmd)
time.sleep(60)
logical_pool = "curve-tool -cluster_map=topo.json -mds_addr=%s\
-op=create_logicalpool" % (mds_addrs)
rs = shell_operator.ssh_exec(ssh, logical_pool)
time.sleep(180)
def check_fuse_mount_success(fs_mount_dir=config.fs_mount_dir):
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
for mount_dir in fs_mount_dir:
grep_cmd = "ps -ef | grep curve-fuse | grep failover/%s | grep -v grep | awk '{print $2}' " % mount_dir
rs = shell_operator.ssh_exec(ssh, grep_cmd)
pid = ""
if rs[1] != []:
pid = rs[1][0].strip()
logger.info("pid=%s" % pid)
if pid:
logger.debug("process is: %s" % pid)
try:
ori_cmd = "stat -c %a " + "%s/%s" % (config.fs_mount_path,
mount_dir)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "stat dir %s fail,error is %s" % (
mount_dir, rs[1])
permission = "".join(rs[1]).strip()
assert permission == "1777", "dir mount fail,permission is %s" % (
mount_dir, permission)
except Exception:
logger.error(" mount test dir fail.")
raise
else:
assert False, "process %s not exsits" % mount_dir
ssh.close()
def check_fuse_iops(limit=1):
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
port_list = []
for mnt in config.fs_mount_dir:
ori_cmd = "ps -ef|grep failover/%s | grep curve-fuse | grep -v grep | awk '{print $2}'" % mnt
rs = shell_operator.ssh_exec(ssh, ori_cmd)
pid = "".join(rs[1]).strip()
ori_cmd = "sudo netstat -lntp |grep %s |awk '{print $4}'" % pid
rs = shell_operator.ssh_exec(ssh, ori_cmd)
port_list.append("".join(rs[1]).strip())
for port in port_list:
# port = port.strip()
logger.info("get port %s ops" % port)
ori_cmd = "sudo curl -s http://" + port + "/vars" + " | grep \'user_write_bps :\'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "get bps fail,rs is %s" % rs[1]
write_bps = "".join(rs[1]).strip().split(":")[-1]
logger.info("now port %s bps is %s" % (port, write_bps))
if write_bps.isdigit():
assert int(
write_bps
) > limit, "get port %s user_write_bps %s is lower than %d" % (
port, write_bps, limit)
def start_abnormal_test_services():
try:
for host in config.etcd_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo rm -rf /etcd/default.etcd"
shell_operator.ssh_exec(ssh, ori_cmd)
etcd_cmd = "cd etcdrun && sudo nohup ./run.sh new &"
shell_operator.ssh_background_exec2(ssh, etcd_cmd)
ori_cmd = "ps -ef|grep -v grep | grep -w etcd | awk '{print $2}'"
time.sleep(2)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
logger.debug("etcd pid is %s"%rs[1])
assert rs[1] != [], "up etcd fail"
for host in config.mds_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
mds_cmd = "sudo nohup /usr/bin/curve-mds --confPath=/etc/curve/mds.conf &"
shell_operator.ssh_background_exec2(ssh, mds_cmd)
time.sleep(1)
ori_cmd = "ps -ef|grep -v grep | grep -v curve-mds.log | grep -v sudo | grep -w curve-mds | awk '{print $2}'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] != [], "up mds fail"
logger.debug("mds pid is %s"%rs[1])
for host in config.snap_server_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "cd snapshot/temp && sudo nohup curve-snapshotcloneserver -conf=/etc/curve/snapshot_clone_server.conf &"
shell_operator.ssh_background_exec2(ssh, ori_cmd)
except Exception:
logger.error("up servers fail.")
raise
def check_vdbench_output():
try:
ssh = shell_operator.create_ssh_connect(config.fs_test_client[0], 1046,
config.abnormal_user)
ori_cmd = "grep \"Vdbench execution completed successfully\" /home/nbs/tools/vdbench/output -R"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] == []:
t = time.time()
ori_cmd = "mv /home/nbs/tools/vdbench/output /home/nbs/vdbench-output/output-%d" % int(
t)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ori_cmd = "cat /home/nbs/tools/vdbench/output/errorlog.html | grep 'Corrupted data block'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] != []:
error_mes = rs[1][0].strip()
error_file = re.search(r'(?<=file=).*(?=;)', error_mes).group()
ori_cmd = "sudo stat %s > /home/nbs/vdbench-output/output-%d/inode" % (
error_file, int(t))
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert False, "vdbench test error,save log to test client /home/nbs/vdbench-output/output-%d" % int(
t)
except Exception as e:
ssh.close()
raise
ssh.close()
def check_fs_io_error():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo grep \'error\' /var/log/kern.log -R | grep -v libpthread"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] != []:
ori_cmd = "sudo logrotate -vf /etc/logrotate.d/rsyslog"
shell_operator.ssh_exec(ssh, ori_cmd)
assert False," rwio error,log is %s"%rs[1]
ssh.close()
def restart_cinder_server():
for client_host in config.client_list:
ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
ori_cmd = "sudo cp /usr/curvefs/curvefs.py /srv/stack/cinder/lib/python2.7/site-packages/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ori_cmd = "sudo cp /usr/curvefs/_curvefs.so /srv/stack/cinder/lib/python2.7/site-packages/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
time.sleep(2)
ori_cmd = "sudo service cinder-volume restart"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] == [],"rs is %s"%rs
def attach_snapshot_vol(vol_uuid):
ori_cmd = "source OPENRC && nova list |grep %s | awk '{print $2}'"%config.vm_host
ssh = shell_operator.create_ssh_connect(config.client_list[0], 1046, config.abnormal_user)
rs = shell_operator.ssh_exec(ssh,ori_cmd,logger2)
vm_uuid = "".join(rs[1]).strip()
ori_cmd = "source OPENRC &&nova volume-attach %s %s"%((vm_uuid,vol_uuid))
rs = shell_operator.ssh_exec(ssh,ori_cmd,logger2)
logger2.info("exec cmd %s" % ori_cmd)
assert rs[3] == 0,"detach vol fail,return is %s"%rs[2]
logger2.info("exec cmd %s"%ori_cmd)
ssh.close()
def check_test_dir_file_md5():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
test_dir = os.path.join(config.fs_mount_path,config.fs_mount_dir[1])
ori_cmd ="cd " + test_dir + " && find vdb.1_1.dir/ -name 'vdb_*' -type f -print0 | xargs -0 md5sum > md5_2"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"get file md5_2 error, output %s"%rs[1]
ori_cmd = "cd " + test_dir + " && diff md5_1 md5_2"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"diff md5 file fail, output %s"%rs[1]
assert rs[1] == [],"check fio test dir file md5 fail,diff is %s"%rs[1]
def nbd_unmap(self):
if self.dev != "":
cmd = "sudo curve-nbd unmap %s"%(self.dev)
rs = shell_operator.ssh_exec(self.ssh, cmd)
assert rs[3] == 0,"unmap fail：%s"%rs[1]
logger3.info("unmap dev %s"%self.dev)
cmd = "sudo curve-nbd list-mapped |grep %s_ | awk '{print $3}'"%(self.name)
rs = shell_operator.ssh_exec(self.ssh, cmd)
assert "".join(rs[1]).strip() == "","unmap fail,map dev is %s"%rs[1]
else:
assert False,"can not find nbd"
def add_config_file():
for host in config.mds_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo cp -r /etc/curve-bak /etc/curve"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"add host %s config fail,error is %s"%(host,rs[2])
for host in config.chunkserver_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo cp -r /etc/curve-bak /etc/curve"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"add host %s config fail,error is %s"%(host,rs[2])
def check_vdbench_consistency():
try:
ssh = shell_operator.create_ssh_connect(config.fs_test_client[0], 1046, config.abnormal_user)
ori_cmd = "sed -i 's#format=.*#format=no#g' /home/nbs/tools/vdbench/profile"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
t = time.time()
ori_cmd = "cd /home/nbs/tools/vdbench && sudo ./vdbench -jr -f profile -o /home/nbs/vdbench-output/umount-output-%d"%int(t)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"inconsistent after umount vdbench,output is in /home/nbs/vdbench-output/umount-output-%d"%int(t)
except Exception as e:
raise
ssh.close()
def remk_test_dir():
try:
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
for test_dir in config.fs_mount_dir:
ori_cmd = "rm -rf %s/%s"%(config.fs_mount_path,test_dir)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"rm test dir %s fail,error is %s"%(test_dir,rs[1])
ori_cmd = "mkdir %s/%s"%(config.fs_mount_path,test_dir)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mkdir %s fail,error is %s"%(test_dir,rs[1])
except Exception:
logger.error(" remk test dir fail.")
raise
def destroy_curvefs():
try:
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
for mountpoint in config.fs_mount_dir:
cmd = "sudo /home/nbs/.curveadm/bin/curveadm umount %s%s"%(config.fs_mount_path,mountpoint)
shell_operator.ssh_exec(ssh, cmd)
cmd = "/home/nbs/.curveadm/bin/curveadm stop"
ret = shell_operator.run_exec(cmd)
cmd = "echo 'y' | /home/nbs/.curveadm/bin/curveadm clean"
ret = shell_operator.run_exec(cmd)
except Exception:
logger.error("destroy curvefs fail.")
raise
def start_fs_fio():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo supervisorctl stop all && sudo supervisorctl reload"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
time.sleep(5)
ssh.close()
def nbd_map(self,dev=None):
if dev:
cmd = "sudo curve-nbd map cbd:pool1//%s_%s_ --device /dev/%s >> /data/log/curve/curve-nbd.console.log 2>&1"%(self.name,self.user,dev)
else:
cmd = "sudo curve-nbd map cbd:pool1//%s_%s_ >> /data/log/curve/curve-nbd.console.log 2>&1"%(self.name,self.user)
rs = shell_operator.ssh_exec(self.ssh, cmd)
assert rs[3] == 0,"map nbd fail：%s"%rs[1]
def clean_fs_kernel_log():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo logrotate -vf /etc/logrotate.d/rsyslog"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0," rollback log fail, %s"%rs[1]
ssh.close()
def check_corefile():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "sudo ls /corefile |grep core"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[1] != []:
assert False,"/corefile have coredump file,is %s"%rs[1]
def clean_corefile():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
ori_cmd = "sudo mv /corefile/core* /corefile/backup/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ssh.close()
def umount_test_dir(mountpoint=""):
try:
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
if mountpoint == "":
for mountpoint in config.fs_mount_dir:
cmd = "sudo /home/nbs/.curveadm/bin/curveadm umount %s%s"%(config.fs_mount_path,mountpoint)
rs = shell_operator.ssh_exec(ssh, cmd)
assert rs[3] == 0,"umount %s dir fail,error is %s"%(mountpoint,rs[2])
else:
cmd = "sudo /home/nbs/.curveadm/bin/curveadm umount %s%s"%(config.fs_mount_path,mountpoint)
rs = shell_operator.ssh_exec(ssh, cmd)
assert rs[3] == 0,"umount %s dir fail,error is %s"%(mountpoint,rs[2])
except Exception:
logger.error("umount dir fail.")
raise
def nbd_getdev(self):
cmd = "sudo curve-nbd list-mapped |grep %s_ | awk '{print $3}'" % (
self.name)
rs = shell_operator.ssh_exec(self.ssh, cmd)
self.dev = "".join(rs[1]).strip()
logger3.info("map %s to %s" % (self.name, self.dev))
assert rs[3] == 0, "map fail：%s" % rs[1]
def vol_write_data():
ssh = shell_operator.create_ssh_connect(config.vm_stability_host, 22, config.vm_user)
start_time = time.time()
while time.time() - start_time < 120:
ori_cmd = "lsblk |grep %dG | awk '{print $1}'"%config.snapshot_size
rs = shell_operator.ssh_exec(ssh, ori_cmd, logger2)
vd = "".join(rs[1]).strip()
if vd != "":
break
time.sleep(5)
if vd != "":
logger2.info("vd is %s"%vd)
ori_cmd = "fio -name=/dev/%s -direct=1 -iodepth=8 -rw=write -ioengine=libaio -bs=1024k -size=%dG -numjobs=1 -time_based"%(vd,config.snapshot_size)
rs = shell_operator.ssh_exec(ssh, ori_cmd, logger2)
assert rs[3] == 0,"write fio fail"
else:
assert False,"get vd fail"
def get_fuse_pid(mount_dir):
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
grep_cmd = "ps -ef | grep curve-fuse | grep %s | grep -v grep | awk '{print $2}' " % mount_dir
rs = shell_operator.ssh_exec(ssh,grep_cmd)
pid = "".join(rs[1]).strip()
logger.info("pid=%s" %pid)
return pid
def install_deb():
try:
# mkdeb_url = config.curve_workspace + "mk-deb.sh"
# exec_mkdeb = "bash %s"%mkdeb_url
# shell_operator.run_exec2(exec_mkdeb)
cmd = "ls %scurve-mds*.deb" % config.curve_workspace
mds_deb = shell_operator.run_exec2(cmd)
version = mds_deb.split('+')[1]
for host in config.mds_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite *%s* aws-sdk_1.0_amd64.deb" % version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "mds install deb fail,error is %s %s" % (rs[1],
rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
for host in config.client_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite curve-sdk*%s*" % version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "sdk install deb fail,error is %s %s" % (rs[1],
rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
for host in config.chunkserver_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/" %\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite curve-chunkserver*%s* curve-tools*%s* aws-sdk_1.0_amd64.deb" % (
version, version)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[
3] == 0, "chunkserver install deb fail,error is %s %s" % (
rs[1], rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
except Exception:
logger.error("install deb fail.")
raise
def stop_nebd():
for host in config.client_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "ps -ef|grep -v grep | grep nebd | awk '{print $2}' | sudo xargs kill -9"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[3] != 0:
logger.debug("snapshotcloneserver not up")
continue
def destroy_curvefs():
try:
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
cmd = "/home/nbs/.curveadm/bin/curveadm cluster checkout citest"
ret = shell_operator.run_exec(cmd)
assert ret == 0 ,"checkout fail"
for mountpoint in config.fs_mount_dir:
cmd = "sudo /home/nbs/.curveadm/bin/curveadm umount %s%s"%(config.fs_mount_path,mountpoint)
shell_operator.ssh_exec(ssh, cmd)
cmd = "echo 'yes' | /home/nbs/.curveadm/bin/curveadm stop"
ret = shell_operator.run_exec(cmd)
cmd = "echo 'yes' | /home/nbs/.curveadm/bin/curveadm clean --only='data,container'"
ret = shell_operator.run_exec(cmd)
except Exception:
logger.error("destroy curvefs fail.")
raise
def start_nebd():
cmd = "ls nebd/nebd*.deb"
nebd_deb = shell_operator.run_exec2(cmd)
version = nebd_deb.split('+')[1]
assert nebd_deb != "","can not get nebd deb"
for host in config.client_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %snebd/*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite nebd_*%s"%version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"install nebd deb fail,error is %s"%rs
rm_deb = "rm nebd_*%s"%version
shell_operator.ssh_exec(ssh, rm_deb)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 nebd/etc/nebd/*.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = "sudo cp nebd-client.conf nebd-server.conf /etc/nebd/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"cp %s nebd conf fail"%host
ori_cmd = "sudo nebd-daemon start"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[3] != 0:
logger.debug("nebd start fail,error is %s"%rs[1])
ori_cmd == "sudo nebd-daemon restart"
rs2 = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs2[3] == 0,"restart nebd fail, return is %s"%rs2[1]
time.sleep(5)
ori_cmd = "ps -ef|grep nebd-server | grep -v daemon |grep -v grep |awk '{print $2}'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] != "","start nebd fail!"
def nbd_create(self, vol_size):
stripeUnit = [524288, 1048576, 2097152, 4194304]
stripeCount = [1, 2, 4, 8, 16]
unit = random.choice(stripeUnit)
count = random.choice(stripeCount)
cmd = "curve create --filename /%s --length %s --user test --stripeUnit %d --stripeCount %d" % (
self.name, vol_size, unit, count)
rs = shell_operator.ssh_exec(self.ssh, cmd)
assert rs[3] == 0, "create nbd fail：%s" % rs[1]
def wait_op_finish():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
cmd = "sudo supervisorctl stop all"
shell_operator.ssh_exec(ssh, cmd)
ori_cmd1 = "ps -ef|grep -v grep | grep fio"
ori_cmd2 = "ps -ef|grep -v grep | grep vdbench"
starttime = time.time()
while time.time() - starttime < 7200:
rs1 = shell_operator.ssh_exec(ssh, ori_cmd1)
rs2 = shell_operator.ssh_exec(ssh, ori_cmd2)
if rs1[1] == [] and rs2[1] == []:
return True
else:
logger.debug("fio & vdbench is running")
time.sleep(30)
assert False,"vdbench and fio is running timeout,pid is %s,%s"%(rs1[1],rs2[1])
ssh.close()
def check_fs_copyset_status():
mds = config.fs_mds[0]
ssh = shell_operator.create_ssh_connect(mds, 1046, config.abnormal_user)
ori_cmd = "sudo docker ps |grep curvefs | awk '{print $1}'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
docker_id = rs[1][0].strip()
logger.info("docker is %s" % rs[1])
ori_cmd = "sudo docker exec -i %s curvefs_tool status-copyset |grep unhealth" % docker_id
rs = shell_operator.ssh_exec(ssh, ori_cmd)
logger.info("status is %s" % rs[1])
if rs[0] != 0 and rs[1] == []:
logger.info("copyset is healthy")
return True
else:
logger.info("copyset is unhealthy")
ori_cmd = "sudo docker exec -i %s curvefs_tool status-copyset" % docker_id
rs = shell_operator.ssh_exec(ssh, ori_cmd)
logger.debug("copyset status is %s" % rs[1])
return False
