def start_nebd():
cmd = "ls nebd/nebd*.deb"
nebd_deb = shell_operator.run_exec2(cmd)
version = nebd_deb.split('+')[1]
assert nebd_deb != "","can not get nebd deb"
for host in config.client_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %snebd/*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite nebd_*%s"%version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"install nebd deb fail,error is %s"%rs
rm_deb = "rm nebd_*%s"%version
shell_operator.ssh_exec(ssh, rm_deb)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 nebd/etc/nebd/*.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = "sudo cp nebd-client.conf nebd-server.conf /etc/nebd/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"cp %s nebd conf fail"%host
ori_cmd = "sudo nebd-daemon start"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
if rs[3] != 0:
logger.debug("nebd start fail,error is %s"%rs[1])
ori_cmd == "sudo nebd-daemon restart"
rs2 = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs2[3] == 0,"restart nebd fail, return is %s"%rs2[1]
time.sleep(5)
ori_cmd = "ps -ef|grep nebd-server | grep -v daemon |grep -v grep |awk '{print $2}'"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[1] != "","start nebd fail!"
def start_fs_vdbench():
test_client = config.fs_test_client[0]
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 robot/Resources/config/profile \
%s:~/tools/vdbench/"%(config.pravie_key_path,test_client)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(test_client, 1046, config.abnormal_user)
ori_cmd = "cd /home/nbs/tools/vdbench && sudo nohup ./vdbench -jn -f profile &"
rs = shell_operator.ssh_background_exec2(ssh, ori_cmd)
time.sleep(5)
ssh.close()
def test_fuse_client_cpu_stress(stress=80):
# client_host = random.choice(config.client_list)
client_host = config.fs_test_client[0]
logger.info("|------begin test fuse client cpu stress,host %s------|"%(client_host))
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 robot/Resources/keywords/cpu_stress.py \
%s:~/"%(config.pravie_key_path,client_host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(client_host, 1046, config.abnormal_user)
fault_inject.inject_cpu_stress(ssh,stress)
return ssh
def check_process_exsits(process_name):
grep_cmd = "ps -ef | grep %s | grep -v grep | awk '{print $2}' " % process_name
pid = shell_operator.run_exec2(grep_cmd)
logger.info("pid=%s" %pid)
if pid:
cmd = "ps -ef|grep %s | grep -v grep" % process_name
process = shell_operator.run_exec2(cmd)
logger.debug("process is: %s" % process)
return 0
else:
logger.debug("process %s not exsits" % process_name)
return -1
def rm_copyset_table():
grep_cmd = "rm mysql.log"
try:
pid = shell_operator.run_exec2(grep_cmd)
except Exception:
logger.error("exec %s error" % grep_cmd)
raise
def mv_copyset_table():
grep_cmd = "mv %s %s" % (config.mysql_log, config.curve_workspace)
try:
pid = shell_operator.run_exec2(grep_cmd)
except Exception:
logger.error("exec %s error" % grep_cmd)
raise
def test_out_metaserver_copyset():
test_client = config.fs_test_client[0]
ssh = shell_operator.create_ssh_connect(test_client, 1046,
config.abnormal_user)
ori_cmd = "/home/nbs/.curveadm/bin/curveadm status --role=metaserver |grep metaserver| awk '{print $1}'"
rs = shell_operator.run_exec2(ori_cmd)
logger.info("rs is %s" % rs)
rs = rs.split('\n')
meta_docker = random.choice(rs)
ori_cmd = "/home/nbs/.curveadm/bin/curveadm stop --id=%s" % meta_docker
logger.info("|------begin test out one metaserver,docker %s------|" %
(meta_docker))
try:
rs = shell_operator.run_exec(ori_cmd)
assert rs == 0, "stop metaserver fail"
time.sleep(120)
# begin_num = get_metaserver_copyset_num(chunkserver_host,cs_list[0])
starttime = time.time()
while time.time() - starttime < 1200:
status = check_fs_copyset_status()
if status == True:
break
time.sleep(60)
logger.info("copyset is unhealthy")
assert status == True, "cluster metaserver %s not recover finish in %d" % (
meta_docker, 120)
except Exception as e:
# raise AssertionError()
logger.error("error is %s" % e)
test_start_process("metaserver")
raise
def get_copyset_scatterwith():
cmd1 = 'sed -i "s/\\t//g" mysql.log'
cmd2 = 'sed -i ":label;N;s/\\n//;b label" mysql.log'
try:
shell_operator.run_exec2(cmd1)
shell_operator.run_exec2(cmd2)
except Exception:
logger.error("exec %s %s error" % (cmd1, cmd2))
raise
copyset_list = {}
try:
row = open("mysql.log").read()
except Exception:
logger.error("open mysql.log fail")
raise
row = row.split('|')
for i in range(0, len(row)):
if i % 2 != 0:
copyset_list[(i + 1) / 2] = eval(row[i])
logger.info("copset length is %d" % len(copyset_list))
assert len(copyset_list) == config.copyset_num
cs_list = []
for key, value in copyset_list.items():
cs = value
for i in range(len(cs)):
cs_list.append(cs[i])
# print cs_list
cs_list = list(set(cs_list))
# logger.info("chunkserver list is %s"%cs_list)
for cs in cs_list:
cs_copyset_num = 0
scatterwith = []
for key, value in copyset_list.items():
if cs in value:
cs_copyset_num += 1
for i in range(len(value)):
scatterwith.append(value[i])
scatterwith = list(set(scatterwith))
assert cs_copyset_num == (config.copyset_num * 3 / config.cs_num)
logger.info(
"chunkserver %d copyset_num is %d \t,scatterwith is %d %s" %
(cs, cs_copyset_num, len(scatterwith), scatterwith))
def kill_process(process_name):
grep_cmd = "ps -ef | grep %s | grep -v grep | awk '{print $2}' " % process_name
pids = shell_operator.run_exec2(grep_cmd)
logger.info("pid=%s" % pids)
if pids:
kill_cmd = "ps -ef | grep %s | grep -v grep | awk '{print $2}' | sudo xargs kill -9" % process_name
ret_code = shell_operator.run_exec(kill_cmd)
if ret_code == 0:
return 0
else:
logger.error("kill process fail %s" % process_name)
else:
logger.debug("process %s not start." % process_name)
def install_deb():
try:
# mkdeb_url = config.curve_workspace + "mk-deb.sh"
# exec_mkdeb = "bash %s"%mkdeb_url
# shell_operator.run_exec2(exec_mkdeb)
cmd = "ls %scurve-mds*.deb" % config.curve_workspace
mds_deb = shell_operator.run_exec2(cmd)
version = mds_deb.split('+')[1]
for host in config.mds_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite *%s* aws-sdk_1.0_amd64.deb" % version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "mds install deb fail,error is %s %s" % (rs[1],
rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
for host in config.client_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/"%\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite curve-sdk*%s*" % version
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0, "sdk install deb fail,error is %s %s" % (rs[1],
rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
for host in config.chunkserver_list:
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s*.deb %s:~/" %\
(config.pravie_key_path,config.curve_workspace,host)
shell_operator.run_exec2(cmd)
ssh = shell_operator.create_ssh_connect(host, 1046,
config.abnormal_user)
ori_cmd = "sudo dpkg -i --force-overwrite curve-chunkserver*%s* curve-tools*%s* aws-sdk_1.0_amd64.deb" % (
version, version)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[
3] == 0, "chunkserver install deb fail,error is %s %s" % (
rs[1], rs[2])
rm_deb = "rm *%s*" % version
shell_operator.ssh_exec(ssh, rm_deb)
except Exception:
logger.error("install deb fail.")
raise
def add_config():
etcd = []
for host in config.etcd_list:
etcd.append(host + ":12379")
etcd_addrs = ",".join(etcd)
# add mds config
for host in config.mds_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo rm *.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 conf/mds.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = R"sed -i 's/mds.listen.addr=127.0.0.1:6666/mds.listen.addr=%s:6666/g' mds.conf"%host
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
#change offline time
ori_cmd = R"sed -i 's/mds.heartbeat.offlinetimeoutMs=.*/mds.heartbeat.offlinetimeoutMs=%d/g' mds.conf"%(config.offline_timeout*1000)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
#change clean_follower_afterMs time
ori_cmd = R"sed -i 's/mds.heartbeat.clean_follower_afterMs=.*/mds.heartbeat.clean_follower_afterMs=%d/g' mds.conf"%(300000)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
#change scheduler time
ori_cmd = R"sed -i 's/mds.copyset.scheduler.intervalSec=.*/mds.copyset.scheduler.intervalSec=0/g' mds.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
ori_cmd = R"sed -i 's/mds.replica.scheduler.intervalSec=.*/mds.replica.scheduler.intervalSec=0/g' mds.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
# ori_cmd = R"sed -i 's/mds.recover.scheduler.intervalSec=.*/mds.recover.scheduler.intervalSec=0/g' mds.conf"
# rs = shell_operator.ssh_exec(ssh, ori_cmd)
# assert rs[3] == 0,"change host %s mds config fail"%host
ori_cmd = R"sed -i 's/mds.leader.scheduler.intervalSec=.*/mds.leader.scheduler.intervalSec=5/g' mds.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
# change topology update time
ori_cmd = R"sed -i 's/mds.topology.TopologyUpdateToRepoSec=.*/mds.topology.TopologyUpdateToRepoSec=1/g' mds.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
#add mysql conf
ori_cmd = R"sed -i 's/mds.DbUrl=localhost/mds.DbUrl=%s/g' mds.conf"%(config.abnormal_db_host)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
#add etcd conf
ori_cmd = R"sed -i 's/mds.etcd.endpoint=127.0.0.1:2379/mds.etcd.endpoint=%s/g' mds.conf"%(etcd_addrs)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s mds config fail"%host
ori_cmd = "sudo mv mds.conf /etc/curve/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv %s mds conf fail"%host
# add client config
mds_addrs = []
for host in config.mds_list:
mds_addrs.append(host + ":6666")
addrs = ",".join(mds_addrs)
for host in config.client_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo rm *.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 conf/client.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = R"sed -i 's/mds.listen.addr=127.0.0.1:6666/mds.listen.addr=%s/g' client.conf"%(addrs)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s client config fail"%host
#将client.conf配置成py_client.conf(主机用)，方便client复现死锁问题
ori_cmd = "sudo mv client.conf /etc/curve/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
ori_cmd = "sudo cp /etc/curve/client.conf /etc/curve/py_client.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv %s client conf fail"%host
# add chunkserver config
addrs = ",".join(mds_addrs)
for host in config.chunkserver_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
ori_cmd = "sudo rm *.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 conf/chunkserver.conf.example %s:~/chunkserver.conf"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
#change global ip
ori_cmd = R"sed -i 's/global.ip=127.0.0.1/global.ip=%s/g' chunkserver.conf"%host
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
#change global subnet
subnet=host+"/24"
ori_cmd = R"sed -i 's#global.subnet=127.0.0.0/24#global.subnet=%s#g' chunkserver.conf"%subnet
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
#change mds ip
ori_cmd = R"sed -i 's/mds.listen.addr=127.0.0.1:6666/mds.listen.addr=%s/g' chunkserver.conf"%(addrs)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = R"sed -i 's/chunkserver.snapshot_throttle_throughput_bytes=.*/chunkserver.snapshot_throttle_throughput_bytes=104857600/g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = R"sed -i 's/trash.expire_afterSec=.*/trash.expire_afterSec=0/g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = R"sed -i 's/trash.scan_periodSec=.*/trash.scan_periodSec=10/g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
#open use snapshot
ori_cmd = R"sed -i 's/clone.disable_curve_client=true/clone.disable_curve_client=false/g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = R"sed -i 's/clone.disable_s3_adapter=true/clone.disable_s3_adapter=false/g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = R"sed -i 's#curve.config_path=conf/cs_client.conf#curve.config_path=/etc/curve/conf/cs_client.conf#g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = R"sed -i 's#s3.config_path=conf/s3.conf#s3.config_path=/etc/curve/conf/s3.conf#g' chunkserver.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s chunkserver config fail"%host
ori_cmd = "sudo mv chunkserver.conf /etc/curve/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv %s chunkserver conf fail"%host
# add s3 and client conf\cs_client conf
client_host = random.choice(config.client_list)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 %s:/etc/curve/client.conf ."%\
(config.pravie_key_path,client_host)
shell_operator.run_exec2(cmd)
for host in config.chunkserver_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 conf/s3.conf client.conf conf/cs_client.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = R"sed -i 's/mds.listen.addr=127.0.0.1:6666/mds.listen.addr=%s/g' cs_client.conf"%(addrs)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s cs_client config fail"%host
ori_cmd = "sudo mv s3.conf /etc/curve/conf && sudo mv client.conf /etc/curve/conf && sudo mv cs_client.conf /etc/curve/conf/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv %s s3 conf fail"%host
for host in config.snap_server_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 conf/s3.conf client.conf conf/snapshot_clone_server.conf conf/snap_client.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = "sed -i \"s/client.config_path=\S*/client.config_path=\/etc\/curve\/snap_client.conf/\" snapshot_clone_server.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s snapshot config fail"%host
#修改snapshot_clone_server.conf etcd配置
ori_cmd = "sed -i \"s/etcd.endpoint=\S*/etcd.endpoint=%s/g\" snapshot_clone_server.conf"%(etcd_addrs)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s snapshot config fail"%host
#修改数据库配置项
ori_cmd = R"sed -i 's/metastore.db_address=\S*/metastore.db_address=%s/g' snapshot_clone_server.conf"%(config.abnormal_db_host)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s snapshot clone server config fail"%host
ori_cmd = "sed -i \"s/s3.config_path=\S*/s3.config_path=\/etc\/curve\/s3.conf/\" snapshot_clone_server.conf"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s snapshot config fail"%host
ori_cmd = "sed -i \"s/server.address=\S*/server.address=%s:5556/g\" snapshot_clone_server.conf"%host
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s snapshot config fail"%host
#change snap_client.conf
ori_cmd = "sed -i \"s/mds.listen.addr=\S*/mds.listen.addr=%s/g\" snap_client.conf"%(addrs)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s snapshot config fail"%host
ori_cmd = "sudo mv snapshot_clone_server.conf /etc/curve/ && sudo mv snap_client.conf /etc/curve/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv %s snapshot_clone_server conf fail"%host
ori_cmd = "sudo mv s3.conf /etc/curve/ && sudo mv client.conf /etc/curve/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
# add tools config
snap_addrs_list = []
for host in config.snap_server_list:
snap_addrs_list.append(host + ":5556")
snap_addrs = ",".join(snap_addrs_list)
for host in config.mds_list:
ssh = shell_operator.create_ssh_connect(host, 1046, config.abnormal_user)
rs = shell_operator.ssh_exec(ssh, ori_cmd)
cmd = "scp -i %s -o StrictHostKeyChecking=no -P 1046 conf/tools.conf %s:~/"%\
(config.pravie_key_path,host)
shell_operator.run_exec2(cmd)
ori_cmd = R"sed -i 's/mdsAddr=127.0.0.1:6666/mdsAddr=%s/g' tools.conf"%addrs
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s tools config fail"%host
ori_cmd = R"sed -i 's/etcdAddr=127.0.0.1:2379/etcdAddr=%s/g' tools.conf"%etcd_addrs
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s tools config fail"%host
ori_cmd = R"sed -i 's/snapshotCloneAddr=127.0.0.1:5555/snapshotCloneAddr=%s/g' tools.conf"%snap_addrs
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"change host %s tools config fail"%host
ori_cmd = "sudo mv tools.conf /etc/curve/"
rs = shell_operator.ssh_exec(ssh, ori_cmd)
assert rs[3] == 0,"mv %s tools conf fail"%host
