def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) osdInfoList = osdStatus.checkOsdStatus(driver) #replicaNum='2' #goToAccountSettings() #changeProfilePhoto() poolManagement.createPool(driver, poolName, poolCapcity) #driver.refresh() #createBlockDeviceImg(poolName=poolName,imgName=imgName, imgSize=imgSize) #driver.refresh() #setSnap(imgName=imgName, snapName=snapName, snapNum=interval) #driver.refresh() #deleteScheduleSnap() #createBlockDeviceSnap(imgName=imgName, snapName=snapName) #poolManagement.createScheduleSnap(driver, imgName, snapScheduleName,interval) #poolManagement.deleteScheduleSnap(driver, snapScheduleName) #createClone(snapName=snapName, cloneName=cloneName) #flattenBlockDeviceClone(cloneName=cloneName) #deleteScheduleSnap() #batchCreateBlockDeviceImg(imgName=imgName, imgSize=imgSize,imgNum=imgNum) #deleteBlockDeviceSnap() #deleteBlockDeviceImg(imgName=imgName) #batchDeleteBlockDeviceImg(imgNum=imgNum) #driver.refresh() #logout() #login(ip1="192.168.28.113",port="3579") #deletePool() driver.refresh() loginlogout.logout(driver)
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) if (driver.find_element_by_id("login-error-msg").is_displayed()): logging.getLogger(caseName).info("case failed") logging.getLogger(caseName).info( "now we have loggin, sleep 5 seconds then logout") sleep(2) loginlogout.accountSetting(caseName, driver, firstName='UserFirst', lastName='UserLast') sleep(1) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) poolManagement.createPool(caseName, driver, 'UITestpool', '10') sleep(2) poolManagement.batchCreateBlockDeviceImg(caseName, driver, 'UITestpool', 'batchImg', '10', '5') sleep(2) #poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') poolManagement.batchDeleteBlockDeviceImg(caseName, driver, '5') sleep(2) poolManagement.deletePool(caseName, driver, 'UITestpool') loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) Setting.addUser(caseName, driver, username="******", passwd="123456", email="*****@*****.**", receiveAlertEmail='1', firstName="abc", lastName="cba", auth="No Access", operation='create') sleep(4) Setting.delUser(caseName, driver, username="******") sleep(4) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) Setting.emailSetting(caseName, driver, enable="1", smtpAddress="smtp.office365.com", smtpRelayPort="587", enableSmtpAuth="1", userName="******", passwd="li", enableSenderAddress="*****@*****.**", subject="event notification", notification="Batch All", operation="save") sleep(1) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) for client in clusterObj.getClients(): base.startRBDIO(caseName, client, imageNum, poolName)
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) sleep(2) loginlogout.changeEnglish(driver) sleep(2) logging.getLogger(caseName).info("add normal node") clusterManagement.addMonitor(caseName, driver, '192.168.29.6') sleep(2) clusterManagement.deleteMonitor(caseName, driver, '192.168.29.6') sleep(2) #osdstatus.checkOsdStatus (driver) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0] clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() #init process logging.getLogger(caseName).info('Init osd process') #clusterObj.initOsdProcess(caseName) clusterObj.createPool(caseName, poolName, pgNumber) poolObj = clusterObj.getPoolByname(caseName, poolName) #############Image operation #print poolObj.pgNumber imageSize = baseSize for client in clusterObj.getClients(): if(IOMode == 'rbd'): imageBaseName = client.gethostName() + 'rbdImg' for i in range(0, imageNum): print imageSize imageName = imageBaseName + str(i) clusterObj.createImg(caseName,size = imageSize, pool = poolName, imageName = imageName) imageSize = utils.add(imageSize, increment) elif(IOMode == 'nbd'): for i in range(0, imageNum): if(i<imageNum/2): imageBaseName = client.gethostName() + 'nbdBlockImg' imageName = imageBaseName + str(i) else: imageBaseName = client.gethostName() + 'nbdFileImg' imageName = imageBaseName + str(i-imageNum/2) clusterObj.createImg(caseName,size = imageSize, pool = poolName, imageName = imageName) imageSize = utils.add(imageSize, increment) logging.getLogger(caseName).info('Init config complete')
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) sleep(2) loginlogout.changeEnglish(driver) sleep(2) logging.getLogger(caseName).info("create the pool with name UITestpool") poolManagement.createPool(caseName, driver, 'UITestpool', '10') driver.refresh() sleep(2) logging.getLogger(caseName).info( "create the image with invalid size 01TB ") poolManagement.createBlockDeviceImg(caseName, driver, 'UITestpool', 'UITestImg', '1', imgUnit='TB') driver.refresh() #poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') poolManagement.createBlockDeviceSnap(caseName, driver, 'UITestImg', 'UITestSnap') driver.refresh() sleep(2) try: poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') except: logging.getLogger(caseName).info("The image has snapshot") sleep(2) poolManagement.deleteSnap(caseName, driver, 'UITestSnap') sleep(2) #poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') try: poolManagement.deletePool(caseName, driver, 'UITestpool') except: logging.getLogger(caseName).info("The pool has image") sleep(2) poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') sleep(2) poolManagement.deletePool(caseName, driver, 'UITestpool') loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout")
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] clusterObj = base.getClusterObj(caseName, args) for i in range(0, imageNum): imageName = imageBaseName + str(i) clusterObj.removeImg(caseName, poolName, imageName) clusterObj.removePool(caseName, poolName) logging.getLogger(caseName).info('Deconfig complete')
def main(args): caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() #client = clusterObj.getClients()[0] logging.getLogger(caseName).info("start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) logging.getLogger(caseName).info("\nStep 2: Out the osd and check IO") for nodeObj in nodeList: for osdObj in nodeObj.getOsds(): #out the osd for i in range(5): logging.getLogger(caseName).info("\nNow operate "+nodeObj.gethostName()) logging.getLogger(caseName).info(len(nodeObj.getOsds())) logging.getLogger(caseName).info("\nNow operate "+osdObj.getid()) logging.getLogger(caseName).info("out "+osdObj.getid()) osdObj.outCluster(caseName, nodeObj) logging.getLogger(caseName).info("check if IO error") sleep(15) #add osd in cluster #logging.getLogger(caseName).info("add in "+osdObj.getid()+" to cluster") osdObj.inCluster(caseName, nodeObj) #check ceph health sleep(15) logging.getLogger(caseName).info("Now check if there is any IO error") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("%s runs complete"%caseName)
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) sleep(2) loginlogout.changeEnglish(driver) sleep(2) poolManagement.createPool(caseName, driver, 'UITestpool', '10', replicaSize='3') sleep(2) poolManagement.createBlockDeviceImg(caseName, driver, 'UITestpool', 'UITestImg', '5') sleep(2) poolManagement.createScheduleSnap(caseName, driver, 'UITestImg', 'UITestSnapSchedule', '2') sleep(2) poolManagement.createScheduleSnap(caseName, driver, 'UITestImg', 'UITestSnapScheduleInvalid', '-2') sleep(2) poolManagement.createScheduleSnap(caseName, driver, 'UITestImg', 'UITestSnapScheduleInvalid-', '-2') try: poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') except: logging.getLogger(caseName).info( "delete the block device fail due to snapshot on it") sleep(2) poolManagement.deleteScheduleSnap(caseName, driver, 'UITestSnapSchedule') sleep(2) poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') sleep(2) poolManagement.deletePool(caseName, driver, 'UITestpool') loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) #client = clusterObj.getClients()[0] nodeObj = clusterObj.getFirstAvaNode(caseName) logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) logging.getLogger(caseName).info("\nStep2: pause all osds") clusterObj.pauseOsd(caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("pause cluster successfully") else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("print log for another 10 minutes") status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("resume cluster successfully") else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("\nStep3: resume all osds") clusterObj.resumeOsd(caseName) for client in clusterObj.getClients(): client.checkIOError(caseName) ''' for client in clusterObj.getClients(): base.stopIO(caseName, client) ''' logging.getLogger(caseName).info("\nCase runs successfully")
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) sleep(2) poolManagement.createPool(caseName, driver, 'UITestpool', '10') sleep(2) logging.getLogger(caseName).info("invalid pool") driver.refresh() ''' if(driver.find_element_by_xpath("//div[contains(@class,'modal-content')]").is_displayed()): logging.getLogger(caseName).info("invalid pool") driver.find_element_by_xpath("//a[@id='CANCEL_BTN']").click() ''' logging.getLogger(caseName).info("create image with same name") poolManagement.createBlockDeviceImg(caseName, driver, 'UITestpool', 'UITestImg', '10', imgUnit='TB') sleep(5) poolManagement.createBlockDeviceImg(caseName, driver, 'UITestpool', 'UITestImg', '1', imgUnit='TB') sleep(5) poolManagement.deleteBlockDeviceImg(caseName, driver, 'UITestImg') sleep(2) poolManagement.deletePool(caseName, driver, 'UITestpool') loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) sleep(2) loginlogout.changeEnglish(driver) sleep(2) logging.getLogger(caseName).info("filter audit log with pre to now") #clusterManagement.filterEventLog(driver, preTimeNum="6", preTimeUnit="Hours", source='denali2', service="cassandra", level="High", eventType="cpu-average") clusterManagement.filterAuditLog(caseName, driver, preTimeNum="6", preTimeUnit="Hours", firstName='admin', lastName='admin') sleep(10) logging.getLogger(caseName).info("filter audit log with from to time") curTime = time.time() fromTime = time.strftime('%m/%d/%Y', time.localtime(curTime)) toTime = time.strftime('%m/%d/%Y', time.localtime(curTime + curTime % 86400)) clusterManagement.filterAuditLog(caseName, driver, fromTime=fromTime, toTime=toTime) sleep(2) #osdstatus.checkOsdStatus (driver) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options = driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1 = nodeList[0].getIpAddress() url = "http://" + ip1 + ":" + port userName = nodeList[0].getUserName() passwd = nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1, port, userName, passwd) Setting.enableLDAP( caseName, driver, enabled='1', host='10.0.10.42', port='389', ssl='1', bindDN='CN=qa_bind,OU=LDAP,OU=Istuary,DC=istuary,DC=co', bindDNpasswd='ist.1234', baseDN='OU=Istuary,DC=istuary,DC=co', userID='sAMAccountName', searchCon='(&(objectCategory=Person)(objectClass=person))', firstName='givenName', lastName='sn', userEmail='mail', operation='save') sleep(1) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() options=driverSet.driverSet() driver = webdriver.Chrome(chrome_options=options) ip1=nodeList[0].getIpAddress() url = "http://"+ip1+":"+port userName= nodeList[0].getUserName() passwd=nodeList[0].getPassword() driver.get(url) driver.maximize_window() sleep(2) loginlogout.login(caseName, driver, ip1,port, userName, passwd) Setting.setGeneralTimeout(caseName, driver, timeNum='60', timeUnit='hours') sleep(1) loginlogout.logout(caseName, driver) logging.getLogger(caseName).info("logout") driver.close()
def main(args): caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) logging.getLogger(caseName).info("start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) #client = clusterObj.getClients()[0] nodeObj = clusterObj.getFirstAvaNode(caseName) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) logging.getLogger(caseName).info("\nStep2: kill leader mon 10 times") leaderMonFir = clusterObj.getLeaderMon() leaderMonSec = clusterObj.getLeaderMon() leaderMonFir.shutdown(caseName) leaderMonFir.start(caseName) leaderMonSec.shutdown(caseName) leaderMonSec.start(caseName) for i in range(10): leaderMonFir.setMonPid(caseName) leaderMonFir.forceKill(caseName) for client in clusterObj.getClients(): client.checkIOError(caseName) #TBD: add try sleep(30) #check monitor quorum status leaderId = leaderMonFir.getQuorumLeader(caseName) logging.getLogger(caseName).info("now the leader mon is %s"%leaderId) clusterObj.setLeaderMon(leaderId) leaderMonSec.setMonPid(caseName) leaderMonSec.forceKill(caseName) for client in clusterObj.getClients(): client.checkIOError(caseName) #TBD: add try sleep(30) #check monitor quorum status leaderId = leaderMonSec.getQuorumLeader(caseName) logging.getLogger(caseName).info("now the leader mon is %s"%leaderId) clusterObj.setLeaderMon(leaderId) #start leader second mon again leaderMonSec.start(caseName) leaderMonSec.checkIfMonStart(caseName) #TBD: add try sleep(60) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop mon service on %s in cluster successfully"%nodeObj.gethostName()) else: logging.getLogger(caseName).error("status is %s"%status) logging.getLogger(caseName).error("%s runs failed"%caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed"%caseName) exit(-1) #check IO status for client in clusterObj.getClients(): client.checkIOError(caseName) #check monitor quorum status leaderId = leaderMonSec.getQuorumLeader(caseName) logging.getLogger(caseName).info("now the leader mon is %s"%leaderId) if(leaderId == leaderMonSec.gethostName()): logging.getLogger(caseName).info("%s is back"%leaderId) else: logging.getLogger(caseName).error("leader monitor %s is not back"%leaderId) exit(-1) clusterObj.setLeaderMon(leaderId) #start leader first mon again leaderMonFir.start(caseName) leaderMonFir.checkIfMonStart(caseName) #TBD: add try sleep(60) for client in clusterObj.getClients(): client.checkIOError(caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop mon service on %s in cluster successfully"%nodeObj.gethostName()) else: logging.getLogger(caseName).error("status is %s"%status) logging.getLogger(caseName).error("%s runs failed"%caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed"%caseName) exit(-1) #check IO status for client in clusterObj.getClients(): client.checkIOError(caseName) #check monitor quorum status leaderId = leaderMonFir.getQuorumLeader(caseName) logging.getLogger(caseName).info("now the leader mon is %s"%leaderId) if(leaderId == leaderMonFir.gethostName()): logging.getLogger(caseName).info("%s is back"%leaderId) else: logging.getLogger(caseName).error("leader monitor %s is not back"%leaderId) exit(-1) clusterObj.setLeaderMon(leaderId) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("case runs complete")
def main(args): caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) #client = clusterObj.getClients()[0] nodeObj = clusterObj.getFirstAvaNode(caseName) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) nonLeaderMon = clusterObj.getFirstNonLeaderMon() nonLeaderMon.shutdown(caseName) nonLeaderMon.start(caseName) logging.getLogger(caseName).info("\nStep2: kill non-leader mon 10 times") for i in range(10): nonLeaderMon.setMonPid(caseName) nonLeaderMon.forceKill(caseName) sleep(30) for client in clusterObj.getClients(): client.checkIOError(caseName) #check monitor quorum status leaderIdBefore = nonLeaderMon.getQuorumLeader(caseName) logging.getLogger(caseName).info("now deal the non-leader mon %s"%leaderIdBefore) #create another rbd and start IO #clusterObj.createImg(caseName,size = '10G', pool = poolName, imageName = newImageName) #pid = client.writeRbdFio(caseName, newImageName, poolName) ''' pidList = [] pidList.append(pid) if (client.checkIOProcess(caseName, pidList) == 'Error'): logging.getLogger(caseName).error("IO cannot start") exit ''' #start leader mon again nonLeaderMon.start(caseName) nonLeaderMon.checkIfMonStart(caseName) sleep(30) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop mon service on %s in cluster successfully"%nonLeaderMon.gethostName()) else: logging.getLogger(caseName).error("status is %s"%status) logging.getLogger(caseName).error("%s runs failed"%caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed"%caseName) exit(-1) #check IO status for client in clusterObj.getClients(): client.checkIOError(caseName) #check monitor quorum status leaderIdAfter = nonLeaderMon.getQuorumLeader(caseName) logging.getLogger(caseName).info("now the leader mon is %s"%leaderIdAfter) if(leaderIdBefore == leaderIdAfter): logging.getLogger(caseName).info("the leader mon is not impacted") else: logging.getLogger(caseName).error("the leader mon is not the initial one") exit(-1) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("case runs complete")
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] #print utils.add('10GB','10G') #send testenv as argument, then parse it as a object #TBD: change it to arguments #imageDiskDict = {} clusterObj = base.getClusterObj(caseName, args) ''' for client in clusterObj.getClients(): disk = client.mapNbd(caseName, 'reliablityTestPool', 'IO_test') logging.getLogger(caseName).info(disk) ''' ''' for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) ''' ''' for client in clusterObj.getClients(): client.checkIOProcess(caseName ) for client in clusterObj.getClients(): base.stopIO(caseName, client) ''' #client = clusterObj.getClients()[0].checkIOError(caseName) #print len(clusterObj.getNodes()) nodeList = clusterObj.getNodes() logging.getLogger(caseName).info("case 1 start") logging.getLogger(caseName).info(len(nodeList[0].getOsds())) # osdObj = nodeObj.getOsds()[0] disks = [] nodeList[0].setOsdDisk(caseName) for osdObj in nodeList[0].getOsds(): logging.getLogger(caseName).info(osdObj.getid()) disks.append(osdObj.getDisk()) print disks #nodeList[0].setOsdPid(caseName) #for nodeObj in nodeList: #nodeObj.setOsdPid(caseName) ''' monitors = clusterObj.getMonitors() monitors[0].setMonPid(caseName) monitors[0].shutdown(caseName) monitors[0].start(caseName) monitors[0].checkIfMonStart(caseName) ''' ''' node = clusterObj.getFirstAvaNode(caseName) node.uploadScript(caseName) ''' #clusterObj.getStatus(caseName, nodeList[0], timeOut) #clusterObj.initOsdProcess(caseName) ''' clients = clusterObj.getClients() clients[0].writeNbdFileIO(caseName,'/dev/nbd5' , 'ext2') ''' ''' for client in clients: base.startIO(caseName, client) ''' ''' imageSize = baseSize imageDiskDict = {} for i in range(0, imageNum): print imageSize imageName = imageBaseName + str(i) disk = clients[0].mapNbd(caseName, poolName, imageName) imageDiskDict[imageName] = disk clients[0].writeNbdBlockIO(caseName, imageSize, rate, disk) base.stopIO(caseName, clients[0]) for i in range(0, imageNum): print imageSize imageName = imageBaseName + str(i) disk = clients[0].unmapNbd(caseName, imageName, imageDiskDict[imageName]) ''' #pid = clients[0].writeRbdFio(caseName, 'becky', 'becky') #sleep(5) #clients[0].stopFio(caseName, pid) ''' avaiNode = clusterObj.getFirstAvaNode(caseName) #avaiNode.setOsdPid(caseName) avaiNode.setOsdDisk(caseName) osdlist = avaiNode.getOsds() osdlist[0].delete(caseName,avaiNode ) #avaiNode.create(caseName, 'sdb') clusterObj.updateOsds(avaiNode) osdList = clusterObj.getOsds() for osdObj in osdList: print osdObj.getid() ''' #osdlist[1].delete(caseName, avaiNode) #avaiNode.initOsdProcess(caseName) #print avaiNode.gethostName() #osdlist[0].kill(caseName, avaiNode) #osdlist[0].outCluster(caseName, avaiNode) #osdlist[0].shutdown(caseName, avaiNode) #osdlist[0].start(caseName, avaiNode) #osdlist[0].inCluster(caseName, avaiNode) #osdlist[0].inCluster(caseName, avaiNode) #avaiNode.initOsdProcess(caseName) #print osdlist ''' status = clusterObj.getStatus(avaiNode) logging.getLogger(caseName).info(status) clusterObj.createPool(caseName, 'becky_Test', '20') poolObj = clusterObj.getPoolByname(caseName, 'becky_Test') ''' #############Image operation #print poolObj.pgNumber #clusterObj.createImg(caseName,size = '10G', pool = 'becky_Test', imgname = 'testImg') #clusterObj.removePool(caseName, 'becky_Test') return 1
def main(args): caseName = os.path.basename(inspect.getfile(inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() #client = clusterObj.getClients()[0] logging.getLogger(caseName).info("start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep 1: start IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startIO(caseName, client, 'nbd') ''' for client in clusterObj.getClients(): base.startIO(caseName, client, 'nbd') ''' sleep(60) logging.getLogger(caseName).info("\nStep 2: Out the osd and check IO") for nodeObj in nodeList: for osdObj in nodeObj.getOsds(): #out the osd logging.getLogger(caseName).info("\nNow operate "+nodeObj.gethostName()) logging.getLogger(caseName).info(len(nodeObj.getOsds())) logging.getLogger(caseName).info("\nNow operate "+osdObj.getid()) logging.getLogger(caseName).info("out "+osdObj.getid()) osdObj.outCluster(caseName, nodeObj) logging.getLogger(caseName).info("check if IO error") for client in clusterObj.getClients(): client.checkIOError(caseName) #stop osd service logging.getLogger(caseName).info("Set the "+osdObj.getid()+" pid for kill") nodeObj.setOsdPid(caseName) osdObj.inCluster(caseName, nodeObj) #check ceph health sleep(30) for client in clusterObj.getClients(): client.checkIOError(caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop %s in cluster successfully"%osdObj.getid()) else: logging.getLogger(caseName).error("status is %s"%status) logging.getLogger(caseName).error("print log for another 10 minutes") status = clusterObj.getStatus(caseName, nodeObj, timeOut) if(status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop %s in cluster successfully"%osdObj.getid()) break else: logging.getLogger(caseName).error("%s runs failed"%caseName) exit(-1) for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if(client.checkIOProcess(caseName ) == "error"): base.startIO(caseName, client, 'nbd') logging.getLogger(caseName).info("%s runs complete"%caseName)
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) clusterObj.initOsdProcess(caseName) #client = clusterObj.getClients()[0] #nodeObj = clusterObj.getFirstAvaNode(caseName) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) for nodeObj in clusterObj.getNodes(): nodeObj.setOsdPid(caseName) for osdObj in nodeObj.getOsds(): osdObj.forceKill(caseName, nodeObj) for monObj in clusterObj.getMonitors(): monObj.shutdown(caseName) monObj.start(caseName) for monObj in clusterObj.getMonitors(): monObj.setMonPid(caseName) monObj.forceKill(caseName) #TBD:check IO for nodeObj in clusterObj.getNodes(): for osdObj in nodeObj.getOsds(): osdObj.start(caseName, nodeObj) for monObj in clusterObj.getMonitors(): monObj.start(caseName) logging.getLogger(caseName).info("sleep 10 mins to wait cluster recover") sleep(600) for client in clusterObj.getClients(): client.checkIOError(caseName) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) ''' for client in clusterObj.getClients(): base.stopIO(caseName, client) ''' for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("case runs complete")
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() #client = clusterObj.getClients()[0] #stop osd process and start with ceph-osd -i clusterObj.initOsdProcess(caseName) logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) logging.getLogger(caseName).info("\nStep2: kill three osds ") for nodeObj in nodeList: osdObjList = nodeObj.getOsds() #out the osd logging.getLogger(caseName).info("\nNow operate " + nodeObj.gethostName()) #stop osd service #logging.getLogger(caseName).info("Set the "+osdObj.getid()+" pid for kill") nodeObj.setOsdPid(caseName) logging.getLogger(caseName).info("shutdown three osds on node " + nodeObj.gethostName()) osdObjList[0].forceKill(caseName, nodeObj) osdObjList[1].forceKill(caseName, nodeObj) osdObjList[2].forceKill(caseName, nodeObj) #start osd service for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("start osd on node " + nodeObj.gethostName()) osdObjList[0].start(caseName, nodeObj) osdObjList[1].start(caseName, nodeObj) osdObjList[2].start(caseName, nodeObj) returnCode = osdObjList[0].checkIfOsdStart(caseName, nodeObj) tryCount = 0 while (returnCode == 0 and tryCount < 10): returnCode = osdObjList[0].checkIfOsdStart(caseName, nodeObj) tryCount = tryCount + 1 if (tryCount == 10): logging.getLogger(caseName).error("%s cannot start" % osdObjList[0].getid()) returnCode = osdObjList[1].checkIfOsdStart(caseName, nodeObj) tryCount = 0 while (returnCode == 0 and tryCount < 10): returnCode = osdObjList[1].checkIfOsdStart(caseName, nodeObj) tryCount = tryCount + 1 if (tryCount == 10): logging.getLogger(caseName).error("%s cannot starte" % osdObjList[1].getid()) returnCode = osdObjList[2].checkIfOsdStart(caseName, nodeObj) tryCount = 0 while (returnCode == 0 and tryCount < 10): returnCode = osdObjList[2].checkIfOsdStart(caseName, nodeObj) tryCount = tryCount + 1 if (tryCount == 10): logging.getLogger(caseName).error("%s cannot starte" % osdObjList[2].getid()) #check ceph health sleep(30) for client in clusterObj.getClients(): client.checkIOError(caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "stop three osds in cluster successfully") else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("%s runs failed" % caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "kill in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) ''' logging.getLogger(caseName).info("\nStep3: stop IO from clients") logging.getLogger(caseName).info("\nstop IO from clients") #sleep(60) for client in clusterObj.getClients(): base.stopIO(caseName, client) ''' for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("%s runs complete" % caseName)
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) avaiNode = clusterObj.getFirstAvaNode(caseName) #client = clusterObj.getClients()[0] clusterObj.initOsdProcess(caseName) logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) avaiNode.uploadScript(caseName) osdlist = avaiNode.getOsds() for osdObj in osdlist: osdObj.forceKill(caseName, avaiNode) osdObj.userStart(caseName, avaiNode) sleep(60) logging.getLogger(caseName).info( "\nStep2: remove osd and create them 10 times") for i in range(10): avaiNode.setOsdDisk(caseName) disks = [] logging.getLogger(caseName).info("start to delete osd on node %s " % avaiNode.gethostName()) for osdObj in osdlist: disks.append(osdObj.getDisk()) osdObj.delete(caseName, avaiNode) status = clusterObj.getStatus(caseName, avaiNode, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("%s delete succesfully" % osdObj.getid()) else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("%s runs failed" % caseName) status = clusterObj.getStatus(caseName, avaiNode, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "stop in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) for client in clusterObj.getClients(): client.checkIOError(caseName) clusterObj.updateCluster(avaiNode) logging.getLogger(caseName).info( "all osds on node %s delete succesfully" % avaiNode.gethostName()) logging.getLogger(caseName).info("start to create osd on node %s " % avaiNode.gethostName()) for disk in disks: avaiNode.createOsd(caseName, disk) status = clusterObj.getStatus(caseName, avaiNode, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("%s create succesfully" % osdObj.getid()) else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("%s runs failed" % caseName) status = clusterObj.getStatus(caseName, avaiNode, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "stop in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) for client in clusterObj.getClients(): client.checkIOError(caseName) clusterObj.updateCluster(avaiNode) logging.getLogger(caseName).info( "all osd need to create on node %s create succesfully" % avaiNode.gethostName()) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) for client in clusterObj.getClients(): client.checkIOError(caseName) logging.getLogger(caseName).info("case runs complete") return 1
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) #client = clusterObj.getClients()[0] nodeObj = clusterObj.getFirstAvaNode(caseName) for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startIO(caseName, client, 'nbd') sleep(60) monitors = clusterObj.getMonitors() monitors[0].setMonPid(caseName) monitors[0].shutdown(caseName) sleep(30) #TBD:check if io process is still exist ''' if(client.checkIOProcess(caseName, pidList) == 'Error') : logging.getLogger(caseName).error("some process is wrong") ''' monitors[0].start(caseName) monitors[0].checkIfMonStart(caseName) sleep(30) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "stop mon service on %s in cluster successfully" % nodeObj.gethostName()) else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("%s runs failed" % caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("stop in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) #logging.getLogger(caseName).info("\nstop IO from clients") #sleep(60) for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startIO(caseName, client, 'nbd') logging.getLogger(caseName).info("\ncase runs complete")
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) logging.getLogger(caseName).info("the timeout is %d" % timeOut) clusterObj = base.getClusterObj(caseName, args) clusterObj.initOsdProcess(caseName) nodeList = clusterObj.getNodes() #client = clusterObj.getClients()[0] logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep 1: start IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startIO(caseName, client, 'nbd') sleep(60) logging.getLogger(caseName).info("\nStep 2: stop osd and check IO") #logging.getLogger(caseName).info("\n%d"%len(nodeList)) for nodeObj in nodeList: logging.getLogger(caseName).info("\nNow operate osd on %s" % (nodeObj.gethostName())) for osdObj in nodeObj.getOsds(): #out the osd logging.getLogger(caseName).info("\nNow operate " + osdObj.getid()) #stop osd service logging.getLogger(caseName).info("Set the " + osdObj.getid() + " pid for kill") nodeObj.setOsdPid(caseName) logging.getLogger(caseName).info("shutdown " + osdObj.getid() + " by kill") osdObj.shutdown(caseName, nodeObj) for client in clusterObj.getClients(): client.checkIOError(caseName) #start osd service logging.getLogger(caseName).info("start " + osdObj.getid()) osdObj.start(caseName, nodeObj) returnCode = osdObj.checkIfOsdStart(caseName, nodeObj) tryCount = 0 while (returnCode == 0 and tryCount < 10): returnCode = osdObj.checkIfOsdStart(caseName, nodeObj) tryCount = tryCount + 1 if (tryCount == 10): logging.getLogger(caseName).error("%s cannot start" % osdObj.getid()) #check ceph health sleep(30) for client in clusterObj.getClients(): client.checkIOError(caseName) status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "stop %s in cluster successfully" % osdObj.getid()) else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error( "print log for another 10 minutes") status = clusterObj.getStatus(caseName, nodeObj, timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "stop %s in cluster successfully" % osdObj.getid()) break else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startIO(caseName, client, 'nbd') logging.getLogger(caseName).info("%s runs complete" % caseName)
def main(args): caseName = os.path.basename(inspect.getfile( inspect.currentframe())).split('.')[0] logging.getLogger(caseName).info(caseDescription) clusterObj = base.getClusterObj(caseName, args) nodeList = clusterObj.getNodes() #client = clusterObj.getClients()[0] #stop osd process and start with ceph-osd -i clusterObj.initOsdProcess(caseName) logging.getLogger(caseName).info( "start to check cluster status before case running") status = clusterObj.getStatus(caseName, clusterObj.getFirstAvaNode(caseName), timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info("health status is OK") else: logging.getLogger(caseName).error("health status is error") exit(-1) logging.getLogger(caseName).info("\nStep1: Check IO from clients") for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) sleep(60) logging.getLogger(caseName).info("\nStep2: kill osd on two nodes 10 times") for i in range(10): firOsdList = nodeList[0].getOsds() secOsdList = nodeList[1].getOsds() firOsdId = random.randint(0, len(firOsdList) - 1) secOsdId = random.randint(0, len(secOsdList) - 1) firOsdList[firOsdId].shutdown(caseName, nodeList[0]) for client in clusterObj.getClients(): client.checkIOError(caseName) status = clusterObj.getStatus(caseName, nodeList[0], timeOut) if (status == 'HEALTH_OK'): secOsdList[secOsdId].shutdown(caseName, nodeList[1]) for client in clusterObj.getClients(): client.checkIOError(caseName) status = clusterObj.getStatus(caseName, nodeList[0], timeOut) else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "shutdown osd on two nodes successfully") else: logging.getLogger(caseName).error("status is %s" % status) logging.getLogger(caseName).error("%s runs failed" % caseName) status = clusterObj.getStatus(caseName, nodeList[0], timeOut) if (status == 'HEALTH_OK'): logging.getLogger(caseName).info( "kill in cluster successfully") else: logging.getLogger(caseName).error("%s runs failed" % caseName) exit(-1) firOsdList[firOsdId].start(caseName, nodeList[0]) for client in clusterObj.getClients(): client.checkIOError(caseName) secOsdList[secOsdId].start(caseName, nodeList[1]) for client in clusterObj.getClients(): client.checkIOError(caseName) returnCode = firOsdList[firOsdId].checkIfOsdStart( caseName, nodeList[0]) tryCount = 0 while (returnCode == 0 and tryCount < 10): returnCode = firOsdList[firOsdId].checkIfOsdStart( caseName, nodeList[0]) tryCount = tryCount + 1 if (tryCount == 10): logging.getLogger(caseName).error("%s cannot start" % firOsdList[firOsdId].getid()) returnCode = secOsdList[secOsdId].checkIfOsdStart( caseName, nodeList[1]) tryCount = 0 while (returnCode == 0 and tryCount < 10): returnCode = secOsdList[secOsdId].checkIfOsdStart( caseName, nodeList[1]) tryCount = tryCount + 1 if (tryCount == 10): logging.getLogger(caseName).error("%s cannot start" % secOsdList[secOsdId].getid()) for client in clusterObj.getClients(): client.checkIOError(caseName) for client in clusterObj.getClients(): if (client.checkIOProcess(caseName) == "error"): base.startRBDIO(caseName, client, imageNum, poolName) logging.getLogger(caseName).info("%s runs complete" % caseName)