def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
osdInfoList = osdStatus.checkOsdStatus(driver)
#replicaNum='2'
#goToAccountSettings()
#changeProfilePhoto()
poolManagement.createPool(driver, poolName, poolCapcity)
#driver.refresh()
#createBlockDeviceImg(poolName=poolName,imgName=imgName, imgSize=imgSize)
#driver.refresh()
#setSnap(imgName=imgName, snapName=snapName, snapNum=interval)
#driver.refresh()
#deleteScheduleSnap()
#createBlockDeviceSnap(imgName=imgName, snapName=snapName)
#poolManagement.createScheduleSnap(driver, imgName, snapScheduleName,interval)
#poolManagement.deleteScheduleSnap(driver, snapScheduleName)
#createClone(snapName=snapName, cloneName=cloneName)
#flattenBlockDeviceClone(cloneName=cloneName)
#deleteScheduleSnap()
#batchCreateBlockDeviceImg(imgName=imgName, imgSize=imgSize,imgNum=imgNum)
#deleteBlockDeviceSnap()
#deleteBlockDeviceImg(imgName=imgName)
#batchDeleteBlockDeviceImg(imgNum=imgNum)
#driver.refresh()
#logout()
#login(ip1="192.168.28.113",port="3579")
#deletePool()
driver.refresh()
loginlogout.logout(driver)
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
if (driver.find_element_by_id("login-error-msg").is_displayed()):
logging.getLogger(caseName).info("case failed")
logging.getLogger(caseName).info(
"now we have loggin, sleep 5 seconds then logout")
sleep(2)
loginlogout.accountSetting(caseName,
driver,
firstName='UserFirst',
lastName='UserLast')
sleep(1)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
poolManagement.createPool(caseName, driver, 'UITestpool', '10')
sleep(2)
poolManagement.batchCreateBlockDeviceImg(caseName, driver, 'UITestpool',
'batchImg', '10', '5')
sleep(2)
#poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
poolManagement.batchDeleteBlockDeviceImg(caseName, driver, '5')
sleep(2)
poolManagement.deletePool(caseName, driver, 'UITestpool')
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
Setting.addUser(caseName,
driver,
username="******",
passwd="123456",
email="*****@*****.**",
receiveAlertEmail='1',
firstName="abc",
lastName="cba",
auth="No Access",
operation='create')
sleep(4)
Setting.delUser(caseName, driver, username="******")
sleep(4)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
Setting.emailSetting(caseName,
driver,
enable="1",
smtpAddress="smtp.office365.com",
smtpRelayPort="587",
enableSmtpAuth="1",
userName="******",
passwd="li",
enableSenderAddress="*****@*****.**",
subject="event notification",
notification="Batch All",
operation="save")
sleep(1)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
for client in clusterObj.getClients():
base.startRBDIO(caseName, client, imageNum, poolName)
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
sleep(2)
loginlogout.changeEnglish(driver)
sleep(2)
logging.getLogger(caseName).info("add normal node")
clusterManagement.addMonitor(caseName, driver, '192.168.29.6')
sleep(2)
clusterManagement.deleteMonitor(caseName, driver, '192.168.29.6')
sleep(2)
#osdstatus.checkOsdStatus (driver)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0]
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
#init process
logging.getLogger(caseName).info('Init osd process')
#clusterObj.initOsdProcess(caseName)
clusterObj.createPool(caseName, poolName, pgNumber)
poolObj = clusterObj.getPoolByname(caseName, poolName)
#############Image operation
#print poolObj.pgNumber
imageSize = baseSize
for client in clusterObj.getClients():
if(IOMode == 'rbd'):
imageBaseName = client.gethostName() + 'rbdImg'
for i in range(0, imageNum):
print imageSize
imageName = imageBaseName + str(i)
clusterObj.createImg(caseName,size = imageSize, pool = poolName, imageName = imageName)
imageSize = utils.add(imageSize, increment)
elif(IOMode == 'nbd'):
for i in range(0, imageNum):
if(i<imageNum/2):
imageBaseName = client.gethostName() + 'nbdBlockImg'
imageName = imageBaseName + str(i)
else:
imageBaseName = client.gethostName() + 'nbdFileImg'
imageName = imageBaseName + str(i-imageNum/2)
clusterObj.createImg(caseName,size = imageSize, pool = poolName, imageName = imageName)
imageSize = utils.add(imageSize, increment)
logging.getLogger(caseName).info('Init config complete')
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
sleep(2)
loginlogout.changeEnglish(driver)
sleep(2)
logging.getLogger(caseName).info("create the pool with name UITestpool")
poolManagement.createPool(caseName, driver, 'UITestpool', '10')
driver.refresh()
sleep(2)
logging.getLogger(caseName).info(
"create the image with invalid size 01TB ")
poolManagement.createBlockDeviceImg(caseName,
driver,
'UITestpool',
'UITestImg',
'1',
imgUnit='TB')
driver.refresh()
#poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
poolManagement.createBlockDeviceSnap(caseName, driver, 'UITestImg',
'UITestSnap')
driver.refresh()
sleep(2)
try:
poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
except:
logging.getLogger(caseName).info("The image has snapshot")
sleep(2)
poolManagement.deleteSnap(caseName, driver, 'UITestSnap')
sleep(2)
#poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
try:
poolManagement.deletePool(caseName, driver, 'UITestpool')
except:
logging.getLogger(caseName).info("The pool has image")
sleep(2)
poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
sleep(2)
poolManagement.deletePool(caseName, driver, 'UITestpool')
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
clusterObj = base.getClusterObj(caseName, args)
for i in range(0, imageNum):
imageName = imageBaseName + str(i)
clusterObj.removeImg(caseName, poolName, imageName)
clusterObj.removePool(caseName, poolName)
logging.getLogger(caseName).info('Deconfig complete')
def main(args):
caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
#client = clusterObj.getClients()[0]
logging.getLogger(caseName).info("start to check cluster status before case running")
status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
logging.getLogger(caseName).info("\nStep 2: Out the osd and check IO")
for nodeObj in nodeList:
for osdObj in nodeObj.getOsds():
#out the osd
for i in range(5):
logging.getLogger(caseName).info("\nNow operate "+nodeObj.gethostName())
logging.getLogger(caseName).info(len(nodeObj.getOsds()))
logging.getLogger(caseName).info("\nNow operate "+osdObj.getid())
logging.getLogger(caseName).info("out "+osdObj.getid())
osdObj.outCluster(caseName, nodeObj)
logging.getLogger(caseName).info("check if IO error")
sleep(15)
#add osd in cluster
#logging.getLogger(caseName).info("add in "+osdObj.getid()+" to cluster")
osdObj.inCluster(caseName, nodeObj)
#check ceph health
sleep(15)
logging.getLogger(caseName).info("Now check if there is any IO error")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("%s runs complete"%caseName)
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
sleep(2)
loginlogout.changeEnglish(driver)
sleep(2)
poolManagement.createPool(caseName,
driver,
'UITestpool',
'10',
replicaSize='3')
sleep(2)
poolManagement.createBlockDeviceImg(caseName, driver, 'UITestpool',
'UITestImg', '5')
sleep(2)
poolManagement.createScheduleSnap(caseName, driver, 'UITestImg',
'UITestSnapSchedule', '2')
sleep(2)
poolManagement.createScheduleSnap(caseName, driver, 'UITestImg',
'UITestSnapScheduleInvalid', '-2')
sleep(2)
poolManagement.createScheduleSnap(caseName, driver, 'UITestImg',
'UITestSnapScheduleInvalid-', '-2')
try:
poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
except:
logging.getLogger(caseName).info(
"delete the block device fail due to snapshot on it")
sleep(2)
poolManagement.deleteScheduleSnap(caseName, driver, 'UITestSnapSchedule')
sleep(2)
poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
sleep(2)
poolManagement.deletePool(caseName, driver, 'UITestpool')
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
#client = clusterObj.getClients()[0]
nodeObj = clusterObj.getFirstAvaNode(caseName)
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
logging.getLogger(caseName).info("\nStep2: pause all osds")
clusterObj.pauseOsd(caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("pause cluster successfully")
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("print log for another 10 minutes")
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("resume cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed" % caseName)
exit(-1)
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("\nStep3: resume all osds")
clusterObj.resumeOsd(caseName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
'''
for client in clusterObj.getClients():
base.stopIO(caseName, client)
'''
logging.getLogger(caseName).info("\nCase runs successfully")
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
sleep(2)
poolManagement.createPool(caseName, driver, 'UITestpool', '10')
sleep(2)
logging.getLogger(caseName).info("invalid pool")
driver.refresh()
'''
if(driver.find_element_by_xpath("//div[contains(@class,'modal-content')]").is_displayed()):
logging.getLogger(caseName).info("invalid pool")
driver.find_element_by_xpath("//a[@id='CANCEL_BTN']").click()
'''
logging.getLogger(caseName).info("create image with same name")
poolManagement.createBlockDeviceImg(caseName,
driver,
'UITestpool',
'UITestImg',
'10',
imgUnit='TB')
sleep(5)
poolManagement.createBlockDeviceImg(caseName,
driver,
'UITestpool',
'UITestImg',
'1',
imgUnit='TB')
sleep(5)
poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg')
sleep(2)
poolManagement.deletePool(caseName, driver, 'UITestpool')
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
sleep(2)
loginlogout.changeEnglish(driver)
sleep(2)
logging.getLogger(caseName).info("filter audit log with pre to now")
#clusterManagement.filterEventLog(driver, preTimeNum="6", preTimeUnit="Hours", source='denali2', service="cassandra", level="High", eventType="cpu-average")
clusterManagement.filterAuditLog(caseName,
driver,
preTimeNum="6",
preTimeUnit="Hours",
firstName='admin',
lastName='admin')
sleep(10)
logging.getLogger(caseName).info("filter audit log with from to time")
curTime = time.time()
fromTime = time.strftime('%m/%d/%Y', time.localtime(curTime))
toTime = time.strftime('%m/%d/%Y',
time.localtime(curTime + curTime % 86400))
clusterManagement.filterAuditLog(caseName,
driver,
fromTime=fromTime,
toTime=toTime)
sleep(2)
#osdstatus.checkOsdStatus (driver)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options = driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1 = nodeList[0].getIpAddress()
url = "http://" + ip1 + ":" + port
userName = nodeList[0].getUserName()
passwd = nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1, port, userName, passwd)
Setting.enableLDAP(
caseName,
driver,
enabled='1',
host='10.0.10.42',
port='389',
ssl='1',
bindDN='CN=qa_bind,OU=LDAP,OU=Istuary,DC=istuary,DC=co',
bindDNpasswd='ist.1234',
baseDN='OU=Istuary,DC=istuary,DC=co',
userID='sAMAccountName',
searchCon='(&(objectCategory=Person)(objectClass=person))',
firstName='givenName',
lastName='sn',
userEmail='mail',
operation='save')
sleep(1)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
options=driverSet.driverSet()
driver = webdriver.Chrome(chrome_options=options)
ip1=nodeList[0].getIpAddress()
url = "http://"+ip1+":"+port
userName= nodeList[0].getUserName()
passwd=nodeList[0].getPassword()
driver.get(url)
driver.maximize_window()
sleep(2)
loginlogout.login(caseName, driver, ip1,port, userName, passwd)
Setting.setGeneralTimeout(caseName, driver, timeNum='60', timeUnit='hours')
sleep(1)
loginlogout.logout(caseName, driver)
logging.getLogger(caseName).info("logout")
driver.close()
def main(args):
caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
logging.getLogger(caseName).info("start to check cluster status before case running")
status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
#client = clusterObj.getClients()[0]
nodeObj = clusterObj.getFirstAvaNode(caseName)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
logging.getLogger(caseName).info("\nStep2: kill leader mon 10 times")
leaderMonFir = clusterObj.getLeaderMon()
leaderMonSec = clusterObj.getLeaderMon()
leaderMonFir.shutdown(caseName)
leaderMonFir.start(caseName)
leaderMonSec.shutdown(caseName)
leaderMonSec.start(caseName)
for i in range(10):
leaderMonFir.setMonPid(caseName)
leaderMonFir.forceKill(caseName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
#TBD: add try
sleep(30)
#check monitor quorum status
leaderId = leaderMonFir.getQuorumLeader(caseName)
logging.getLogger(caseName).info("now the leader mon is %s"%leaderId)
clusterObj.setLeaderMon(leaderId)
leaderMonSec.setMonPid(caseName)
leaderMonSec.forceKill(caseName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
#TBD: add try
sleep(30)
#check monitor quorum status
leaderId = leaderMonSec.getQuorumLeader(caseName)
logging.getLogger(caseName).info("now the leader mon is %s"%leaderId)
clusterObj.setLeaderMon(leaderId)
#start leader second mon again
leaderMonSec.start(caseName)
leaderMonSec.checkIfMonStart(caseName)
#TBD: add try
sleep(60)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop mon service on %s in cluster successfully"%nodeObj.gethostName())
else:
logging.getLogger(caseName).error("status is %s"%status)
logging.getLogger(caseName).error("%s runs failed"%caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed"%caseName)
exit(-1)
#check IO status
for client in clusterObj.getClients():
client.checkIOError(caseName)
#check monitor quorum status
leaderId = leaderMonSec.getQuorumLeader(caseName)
logging.getLogger(caseName).info("now the leader mon is %s"%leaderId)
if(leaderId == leaderMonSec.gethostName()):
logging.getLogger(caseName).info("%s is back"%leaderId)
else:
logging.getLogger(caseName).error("leader monitor %s is not back"%leaderId)
exit(-1)
clusterObj.setLeaderMon(leaderId)
#start leader first mon again
leaderMonFir.start(caseName)
leaderMonFir.checkIfMonStart(caseName)
#TBD: add try
sleep(60)
for client in clusterObj.getClients():
client.checkIOError(caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop mon service on %s in cluster successfully"%nodeObj.gethostName())
else:
logging.getLogger(caseName).error("status is %s"%status)
logging.getLogger(caseName).error("%s runs failed"%caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed"%caseName)
exit(-1)
#check IO status
for client in clusterObj.getClients():
client.checkIOError(caseName)
#check monitor quorum status
leaderId = leaderMonFir.getQuorumLeader(caseName)
logging.getLogger(caseName).info("now the leader mon is %s"%leaderId)
if(leaderId == leaderMonFir.gethostName()):
logging.getLogger(caseName).info("%s is back"%leaderId)
else:
logging.getLogger(caseName).error("leader monitor %s is not back"%leaderId)
exit(-1)
clusterObj.setLeaderMon(leaderId)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("case runs complete")
def main(args):
caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
#client = clusterObj.getClients()[0]
nodeObj = clusterObj.getFirstAvaNode(caseName)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
nonLeaderMon = clusterObj.getFirstNonLeaderMon()
nonLeaderMon.shutdown(caseName)
nonLeaderMon.start(caseName)
logging.getLogger(caseName).info("\nStep2: kill non-leader mon 10 times")
for i in range(10):
nonLeaderMon.setMonPid(caseName)
nonLeaderMon.forceKill(caseName)
sleep(30)
for client in clusterObj.getClients():
client.checkIOError(caseName)
#check monitor quorum status
leaderIdBefore = nonLeaderMon.getQuorumLeader(caseName)
logging.getLogger(caseName).info("now deal the non-leader mon %s"%leaderIdBefore)
#create another rbd and start IO
#clusterObj.createImg(caseName,size = '10G', pool = poolName, imageName = newImageName)
#pid = client.writeRbdFio(caseName, newImageName, poolName)
'''
pidList = []
pidList.append(pid)
if (client.checkIOProcess(caseName, pidList) == 'Error'):
logging.getLogger(caseName).error("IO cannot start")
exit
'''
#start leader mon again
nonLeaderMon.start(caseName)
nonLeaderMon.checkIfMonStart(caseName)
sleep(30)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop mon service on %s in cluster successfully"%nonLeaderMon.gethostName())
else:
logging.getLogger(caseName).error("status is %s"%status)
logging.getLogger(caseName).error("%s runs failed"%caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed"%caseName)
exit(-1)
#check IO status
for client in clusterObj.getClients():
client.checkIOError(caseName)
#check monitor quorum status
leaderIdAfter = nonLeaderMon.getQuorumLeader(caseName)
logging.getLogger(caseName).info("now the leader mon is %s"%leaderIdAfter)
if(leaderIdBefore == leaderIdAfter):
logging.getLogger(caseName).info("the leader mon is not impacted")
else:
logging.getLogger(caseName).error("the leader mon is not the initial one")
exit(-1)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("case runs complete")
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
#print utils.add('10GB','10G')
#send testenv as argument, then parse it as a object
#TBD: change it to arguments
#imageDiskDict = {}
clusterObj = base.getClusterObj(caseName, args)
'''
for client in clusterObj.getClients():
disk = client.mapNbd(caseName, 'reliablityTestPool', 'IO_test')
logging.getLogger(caseName).info(disk)
'''
'''
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
'''
'''
for client in clusterObj.getClients():
client.checkIOProcess(caseName )
for client in clusterObj.getClients():
base.stopIO(caseName, client)
'''
#client = clusterObj.getClients()[0].checkIOError(caseName)
#print len(clusterObj.getNodes())
nodeList = clusterObj.getNodes()
logging.getLogger(caseName).info("case 1 start")
logging.getLogger(caseName).info(len(nodeList[0].getOsds()))
# osdObj = nodeObj.getOsds()[0]
disks = []
nodeList[0].setOsdDisk(caseName)
for osdObj in nodeList[0].getOsds():
logging.getLogger(caseName).info(osdObj.getid())
disks.append(osdObj.getDisk())
print disks
#nodeList[0].setOsdPid(caseName)
#for nodeObj in nodeList:
#nodeObj.setOsdPid(caseName)
'''
monitors = clusterObj.getMonitors()
monitors[0].setMonPid(caseName)
monitors[0].shutdown(caseName)
monitors[0].start(caseName)
monitors[0].checkIfMonStart(caseName)
'''
'''
node = clusterObj.getFirstAvaNode(caseName)
node.uploadScript(caseName)
'''
#clusterObj.getStatus(caseName, nodeList[0], timeOut)
#clusterObj.initOsdProcess(caseName)
'''
clients = clusterObj.getClients()
clients[0].writeNbdFileIO(caseName,'/dev/nbd5' , 'ext2')
'''
'''
for client in clients:
base.startIO(caseName, client)
'''
'''
imageSize = baseSize
imageDiskDict = {}
for i in range(0, imageNum):
print imageSize
imageName = imageBaseName + str(i)
disk = clients[0].mapNbd(caseName, poolName, imageName)
imageDiskDict[imageName] = disk
clients[0].writeNbdBlockIO(caseName, imageSize, rate, disk)
base.stopIO(caseName, clients[0])
for i in range(0, imageNum):
print imageSize
imageName = imageBaseName + str(i)
disk = clients[0].unmapNbd(caseName, imageName, imageDiskDict[imageName])
'''
#pid = clients[0].writeRbdFio(caseName, 'becky', 'becky')
#sleep(5)
#clients[0].stopFio(caseName, pid)
'''
avaiNode = clusterObj.getFirstAvaNode(caseName)
#avaiNode.setOsdPid(caseName)
avaiNode.setOsdDisk(caseName)
osdlist = avaiNode.getOsds()
osdlist[0].delete(caseName,avaiNode )
#avaiNode.create(caseName, 'sdb')
clusterObj.updateOsds(avaiNode)
osdList = clusterObj.getOsds()
for osdObj in osdList:
print osdObj.getid()
'''
#osdlist[1].delete(caseName, avaiNode)
#avaiNode.initOsdProcess(caseName)
#print avaiNode.gethostName()
#osdlist[0].kill(caseName, avaiNode)
#osdlist[0].outCluster(caseName, avaiNode)
#osdlist[0].shutdown(caseName, avaiNode)
#osdlist[0].start(caseName, avaiNode)
#osdlist[0].inCluster(caseName, avaiNode)
#osdlist[0].inCluster(caseName, avaiNode)
#avaiNode.initOsdProcess(caseName)
#print osdlist
'''
status = clusterObj.getStatus(avaiNode)
logging.getLogger(caseName).info(status)
clusterObj.createPool(caseName, 'becky_Test', '20')
poolObj = clusterObj.getPoolByname(caseName, 'becky_Test')
'''
#############Image operation
#print poolObj.pgNumber
#clusterObj.createImg(caseName,size = '10G', pool = 'becky_Test', imgname = 'testImg')
#clusterObj.removePool(caseName, 'becky_Test')
return 1
def main(args):
caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
#client = clusterObj.getClients()[0]
logging.getLogger(caseName).info("start to check cluster status before case running")
status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep 1: start IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startIO(caseName, client, 'nbd')
'''
for client in clusterObj.getClients():
base.startIO(caseName, client, 'nbd')
'''
sleep(60)
logging.getLogger(caseName).info("\nStep 2: Out the osd and check IO")
for nodeObj in nodeList:
for osdObj in nodeObj.getOsds():
#out the osd
logging.getLogger(caseName).info("\nNow operate "+nodeObj.gethostName())
logging.getLogger(caseName).info(len(nodeObj.getOsds()))
logging.getLogger(caseName).info("\nNow operate "+osdObj.getid())
logging.getLogger(caseName).info("out "+osdObj.getid())
osdObj.outCluster(caseName, nodeObj)
logging.getLogger(caseName).info("check if IO error")
for client in clusterObj.getClients():
client.checkIOError(caseName)
#stop osd service
logging.getLogger(caseName).info("Set the "+osdObj.getid()+" pid for kill")
nodeObj.setOsdPid(caseName)
osdObj.inCluster(caseName, nodeObj)
#check ceph health
sleep(30)
for client in clusterObj.getClients():
client.checkIOError(caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop %s in cluster successfully"%osdObj.getid())
else:
logging.getLogger(caseName).error("status is %s"%status)
logging.getLogger(caseName).error("print log for another 10 minutes")
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if(status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop %s in cluster successfully"%osdObj.getid())
break
else:
logging.getLogger(caseName).error("%s runs failed"%caseName)
exit(-1)
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if(client.checkIOProcess(caseName ) == "error"):
base.startIO(caseName, client, 'nbd')
logging.getLogger(caseName).info("%s runs complete"%caseName)
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
clusterObj.initOsdProcess(caseName)
#client = clusterObj.getClients()[0]
#nodeObj = clusterObj.getFirstAvaNode(caseName)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName,
clusterObj.getFirstAvaNode(caseName),
timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
for nodeObj in clusterObj.getNodes():
nodeObj.setOsdPid(caseName)
for osdObj in nodeObj.getOsds():
osdObj.forceKill(caseName, nodeObj)
for monObj in clusterObj.getMonitors():
monObj.shutdown(caseName)
monObj.start(caseName)
for monObj in clusterObj.getMonitors():
monObj.setMonPid(caseName)
monObj.forceKill(caseName)
#TBD:check IO
for nodeObj in clusterObj.getNodes():
for osdObj in nodeObj.getOsds():
osdObj.start(caseName, nodeObj)
for monObj in clusterObj.getMonitors():
monObj.start(caseName)
logging.getLogger(caseName).info("sleep 10 mins to wait cluster recover")
sleep(600)
for client in clusterObj.getClients():
client.checkIOError(caseName)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
'''
for client in clusterObj.getClients():
base.stopIO(caseName, client)
'''
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("case runs complete")
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
#client = clusterObj.getClients()[0]
#stop osd process and start with ceph-osd -i
clusterObj.initOsdProcess(caseName)
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName,
clusterObj.getFirstAvaNode(caseName),
timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
logging.getLogger(caseName).info("\nStep2: kill three osds ")
for nodeObj in nodeList:
osdObjList = nodeObj.getOsds()
#out the osd
logging.getLogger(caseName).info("\nNow operate " +
nodeObj.gethostName())
#stop osd service
#logging.getLogger(caseName).info("Set the "+osdObj.getid()+" pid for kill")
nodeObj.setOsdPid(caseName)
logging.getLogger(caseName).info("shutdown three osds on node " +
nodeObj.gethostName())
osdObjList[0].forceKill(caseName, nodeObj)
osdObjList[1].forceKill(caseName, nodeObj)
osdObjList[2].forceKill(caseName, nodeObj)
#start osd service
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("start osd on node " +
nodeObj.gethostName())
osdObjList[0].start(caseName, nodeObj)
osdObjList[1].start(caseName, nodeObj)
osdObjList[2].start(caseName, nodeObj)
returnCode = osdObjList[0].checkIfOsdStart(caseName, nodeObj)
tryCount = 0
while (returnCode == 0 and tryCount < 10):
returnCode = osdObjList[0].checkIfOsdStart(caseName, nodeObj)
tryCount = tryCount + 1
if (tryCount == 10):
logging.getLogger(caseName).error("%s cannot start" %
osdObjList[0].getid())
returnCode = osdObjList[1].checkIfOsdStart(caseName, nodeObj)
tryCount = 0
while (returnCode == 0 and tryCount < 10):
returnCode = osdObjList[1].checkIfOsdStart(caseName, nodeObj)
tryCount = tryCount + 1
if (tryCount == 10):
logging.getLogger(caseName).error("%s cannot starte" %
osdObjList[1].getid())
returnCode = osdObjList[2].checkIfOsdStart(caseName, nodeObj)
tryCount = 0
while (returnCode == 0 and tryCount < 10):
returnCode = osdObjList[2].checkIfOsdStart(caseName, nodeObj)
tryCount = tryCount + 1
if (tryCount == 10):
logging.getLogger(caseName).error("%s cannot starte" %
osdObjList[2].getid())
#check ceph health
sleep(30)
for client in clusterObj.getClients():
client.checkIOError(caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"stop three osds in cluster successfully")
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("%s runs failed" % caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"kill in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed" % caseName)
exit(-1)
'''
logging.getLogger(caseName).info("\nStep3: stop IO from clients")
logging.getLogger(caseName).info("\nstop IO from clients")
#sleep(60)
for client in clusterObj.getClients():
base.stopIO(caseName, client)
'''
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("%s runs complete" % caseName)
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
avaiNode = clusterObj.getFirstAvaNode(caseName)
#client = clusterObj.getClients()[0]
clusterObj.initOsdProcess(caseName)
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName,
clusterObj.getFirstAvaNode(caseName),
timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
avaiNode.uploadScript(caseName)
osdlist = avaiNode.getOsds()
for osdObj in osdlist:
osdObj.forceKill(caseName, avaiNode)
osdObj.userStart(caseName, avaiNode)
sleep(60)
logging.getLogger(caseName).info(
"\nStep2: remove osd and create them 10 times")
for i in range(10):
avaiNode.setOsdDisk(caseName)
disks = []
logging.getLogger(caseName).info("start to delete osd on node %s " %
avaiNode.gethostName())
for osdObj in osdlist:
disks.append(osdObj.getDisk())
osdObj.delete(caseName, avaiNode)
status = clusterObj.getStatus(caseName, avaiNode, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("%s delete succesfully" %
osdObj.getid())
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("%s runs failed" % caseName)
status = clusterObj.getStatus(caseName, avaiNode, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"stop in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed" %
caseName)
exit(-1)
for client in clusterObj.getClients():
client.checkIOError(caseName)
clusterObj.updateCluster(avaiNode)
logging.getLogger(caseName).info(
"all osds on node %s delete succesfully" % avaiNode.gethostName())
logging.getLogger(caseName).info("start to create osd on node %s " %
avaiNode.gethostName())
for disk in disks:
avaiNode.createOsd(caseName, disk)
status = clusterObj.getStatus(caseName, avaiNode, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("%s create succesfully" %
osdObj.getid())
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("%s runs failed" % caseName)
status = clusterObj.getStatus(caseName, avaiNode, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"stop in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed" %
caseName)
exit(-1)
for client in clusterObj.getClients():
client.checkIOError(caseName)
clusterObj.updateCluster(avaiNode)
logging.getLogger(caseName).info(
"all osd need to create on node %s create succesfully" %
avaiNode.gethostName())
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
logging.getLogger(caseName).info("case runs complete")
return 1
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName,
clusterObj.getFirstAvaNode(caseName),
timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
#client = clusterObj.getClients()[0]
nodeObj = clusterObj.getFirstAvaNode(caseName)
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startIO(caseName, client, 'nbd')
sleep(60)
monitors = clusterObj.getMonitors()
monitors[0].setMonPid(caseName)
monitors[0].shutdown(caseName)
sleep(30)
#TBD:check if io process is still exist
'''
if(client.checkIOProcess(caseName, pidList) == 'Error') :
logging.getLogger(caseName).error("some process is wrong")
'''
monitors[0].start(caseName)
monitors[0].checkIfMonStart(caseName)
sleep(30)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"stop mon service on %s in cluster successfully" %
nodeObj.gethostName())
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("%s runs failed" % caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("stop in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed" % caseName)
exit(-1)
#logging.getLogger(caseName).info("\nstop IO from clients")
#sleep(60)
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startIO(caseName, client, 'nbd')
logging.getLogger(caseName).info("\ncase runs complete")
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
logging.getLogger(caseName).info("the timeout is %d" % timeOut)
clusterObj = base.getClusterObj(caseName, args)
clusterObj.initOsdProcess(caseName)
nodeList = clusterObj.getNodes()
#client = clusterObj.getClients()[0]
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName,
clusterObj.getFirstAvaNode(caseName),
timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep 1: start IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startIO(caseName, client, 'nbd')
sleep(60)
logging.getLogger(caseName).info("\nStep 2: stop osd and check IO")
#logging.getLogger(caseName).info("\n%d"%len(nodeList))
for nodeObj in nodeList:
logging.getLogger(caseName).info("\nNow operate osd on %s" %
(nodeObj.gethostName()))
for osdObj in nodeObj.getOsds():
#out the osd
logging.getLogger(caseName).info("\nNow operate " + osdObj.getid())
#stop osd service
logging.getLogger(caseName).info("Set the " + osdObj.getid() +
" pid for kill")
nodeObj.setOsdPid(caseName)
logging.getLogger(caseName).info("shutdown " + osdObj.getid() +
" by kill")
osdObj.shutdown(caseName, nodeObj)
for client in clusterObj.getClients():
client.checkIOError(caseName)
#start osd service
logging.getLogger(caseName).info("start " + osdObj.getid())
osdObj.start(caseName, nodeObj)
returnCode = osdObj.checkIfOsdStart(caseName, nodeObj)
tryCount = 0
while (returnCode == 0 and tryCount < 10):
returnCode = osdObj.checkIfOsdStart(caseName, nodeObj)
tryCount = tryCount + 1
if (tryCount == 10):
logging.getLogger(caseName).error("%s cannot start" %
osdObj.getid())
#check ceph health
sleep(30)
for client in clusterObj.getClients():
client.checkIOError(caseName)
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"stop %s in cluster successfully" % osdObj.getid())
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error(
"print log for another 10 minutes")
status = clusterObj.getStatus(caseName, nodeObj, timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"stop %s in cluster successfully" % osdObj.getid())
break
else:
logging.getLogger(caseName).error("%s runs failed" %
caseName)
exit(-1)
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startIO(caseName, client, 'nbd')
logging.getLogger(caseName).info("%s runs complete" % caseName)
def main(args):
caseName = os.path.basename(inspect.getfile(
inspect.currentframe())).split('.')[0]
logging.getLogger(caseName).info(caseDescription)
clusterObj = base.getClusterObj(caseName, args)
nodeList = clusterObj.getNodes()
#client = clusterObj.getClients()[0]
#stop osd process and start with ceph-osd -i
clusterObj.initOsdProcess(caseName)
logging.getLogger(caseName).info(
"start to check cluster status before case running")
status = clusterObj.getStatus(caseName,
clusterObj.getFirstAvaNode(caseName),
timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info("health status is OK")
else:
logging.getLogger(caseName).error("health status is error")
exit(-1)
logging.getLogger(caseName).info("\nStep1: Check IO from clients")
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
sleep(60)
logging.getLogger(caseName).info("\nStep2: kill osd on two nodes 10 times")
for i in range(10):
firOsdList = nodeList[0].getOsds()
secOsdList = nodeList[1].getOsds()
firOsdId = random.randint(0, len(firOsdList) - 1)
secOsdId = random.randint(0, len(secOsdList) - 1)
firOsdList[firOsdId].shutdown(caseName, nodeList[0])
for client in clusterObj.getClients():
client.checkIOError(caseName)
status = clusterObj.getStatus(caseName, nodeList[0], timeOut)
if (status == 'HEALTH_OK'):
secOsdList[secOsdId].shutdown(caseName, nodeList[1])
for client in clusterObj.getClients():
client.checkIOError(caseName)
status = clusterObj.getStatus(caseName, nodeList[0], timeOut)
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("%s runs failed" % caseName)
exit(-1)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"shutdown osd on two nodes successfully")
else:
logging.getLogger(caseName).error("status is %s" % status)
logging.getLogger(caseName).error("%s runs failed" % caseName)
status = clusterObj.getStatus(caseName, nodeList[0], timeOut)
if (status == 'HEALTH_OK'):
logging.getLogger(caseName).info(
"kill in cluster successfully")
else:
logging.getLogger(caseName).error("%s runs failed" % caseName)
exit(-1)
firOsdList[firOsdId].start(caseName, nodeList[0])
for client in clusterObj.getClients():
client.checkIOError(caseName)
secOsdList[secOsdId].start(caseName, nodeList[1])
for client in clusterObj.getClients():
client.checkIOError(caseName)
returnCode = firOsdList[firOsdId].checkIfOsdStart(
caseName, nodeList[0])
tryCount = 0
while (returnCode == 0 and tryCount < 10):
returnCode = firOsdList[firOsdId].checkIfOsdStart(
caseName, nodeList[0])
tryCount = tryCount + 1
if (tryCount == 10):
logging.getLogger(caseName).error("%s cannot start" %
firOsdList[firOsdId].getid())
returnCode = secOsdList[secOsdId].checkIfOsdStart(
caseName, nodeList[1])
tryCount = 0
while (returnCode == 0 and tryCount < 10):
returnCode = secOsdList[secOsdId].checkIfOsdStart(
caseName, nodeList[1])
tryCount = tryCount + 1
if (tryCount == 10):
logging.getLogger(caseName).error("%s cannot start" %
secOsdList[secOsdId].getid())
for client in clusterObj.getClients():
client.checkIOError(caseName)
for client in clusterObj.getClients():
if (client.checkIOProcess(caseName) == "error"):
base.startRBDIO(caseName, client, imageNum, poolName)
logging.getLogger(caseName).info("%s runs complete" % caseName)
