def run_perf_tests(tests, debug=testDebug, mask=testMask, logName=logFileName): """Method to execute cats tests, must be in Performance directory""" logger = TestOutput(stdout=True, debug=0, logName=logName) logger.startSuite(name='ChandlerTestSuite') for paramSet in tests.split(','): try: filenameAndTest = paramSet.split(':') #assume file name and and test name are the same if only one given if len(filenameAndTest) < 2: filenameAndTest.append(filenameAndTest[0]) teststring = 'from tools.cats.Performance.%s import %s' % ( filenameAndTest[0], filenameAndTest[1]) exec(compile(teststring, '', 'exec')) teststring = 'test = %s(name=\'%s\', logger=logger)' % ( filenameAndTest[1], filenameAndTest[1]) exec(compile(teststring, '', 'exec')) test.runTest() if logger.debug == 2: checkRepo(logger) except: logger.traceback() logger.endSuite() if logger.debug == 0: logger.easyReadSummary() else: logger.summary() logger.simpleSummary() logger.tinderOutput() if logger.debug < 2: checkRepo(logger) logger.endSuite() import osaf.framework.scripting as scripting scripting.app_ns().root.Quit()
def run_perf_tests(tests, debug=testDebug, mask=testMask, logName=logFileName): """Method to execute cats tests, must be in Performance directory""" logger = TestOutput(stdout=True, debug=0, logName=logName) logger.startSuite(name='ChandlerTestSuite') for paramSet in tests.split(','): try: filenameAndTest = paramSet.split(':') #assume file name and and test name are the same if only one given if len(filenameAndTest) < 2: filenameAndTest.append(filenameAndTest[0]) teststring = 'from tools.cats.Performance.%s import %s' % (filenameAndTest[0], filenameAndTest[1]) exec(compile(teststring, '', 'exec')) teststring = 'test = %s(name=\'%s\', logger=logger)' % (filenameAndTest[1], filenameAndTest[1]) exec(compile(teststring, '', 'exec')) test.runTest() if logger.debug == 2: checkRepo(logger) except: logger.traceback() logger.endSuite() if logger.debug == 0: logger.easyReadSummary() else: logger.summary() logger.simpleSummary() logger.tinderOutput() if logger.debug < 2: checkRepo(logger) logger.endSuite() import osaf.framework.scripting as scripting scripting.app_ns().root.Quit()
def run_tests(tests, debug=testDebug, mask=testMask, logName=logFileName): """Method to execute cats tests, must be in Functional directory.""" logger = TestOutput(stdout=True, debug=debug, mask=mask, logName=logName) testList = tests.split(',') statusBar = Block.findBlockByName("StatusBar") totalTests = len(tests.split(',')) curTest = 0 if len(testList) < 2: logger.startSuite(name=testList[0].split(':')[0]) else: logger.startSuite(name='ChandlerFunctionalTestSuite') # We'll normally run individual tests with an exception wrapper; # --catch=never will turn it off, so that a debugger can stop on # uncaught exceptions. runner = Globals.options.catch != 'never' and run_test_wrapped or run_test for paramSet in testList: curTest += 1 statusBar.setStatusMessage( "Test %d of %d %s" % (curTest, totalTests, paramSet.split(":")[0])) runner(logger, paramSet) if haltOnFailure and logger.testHasFailed: logger.report(False, 'Suite halted on test failure') logger.testsSkipped = len( tests.split(',')) - (tests.split(',').index(paramSet) + 1) break if logger.debug < 2: checkRepo(logger) logger.endSuite() if logger.debug == 0: logger.easyReadSummary() else: logger.summary() logger.simpleSummary() logger.tinderOutput() scripting.app_ns().root.Quit()
def run_tests(tests, debug=testDebug, mask=testMask, logName=logFileName): """Method to execute cats tests, must be in Functional directory.""" logger = TestOutput(stdout=True, debug=debug, mask=mask, logName=logName) testList = tests.split(',') statusBar = Block.findBlockByName("StatusBar") totalTests = len(tests.split(',')) curTest = 0 if len(testList) < 2: logger.startSuite(name = testList[0].split(':')[0]) else: logger.startSuite(name='ChandlerFunctionalTestSuite') # We'll normally run individual tests with an exception wrapper; # --catch=never will turn it off, so that a debugger can stop on # uncaught exceptions. runner = Globals.options.catch != 'never' and run_test_wrapped or run_test for paramSet in testList: curTest += 1 statusBar.setStatusMessage("Test %d of %d %s" % (curTest, totalTests, paramSet.split(":")[0])) runner(logger, paramSet) if haltOnFailure and logger.testHasFailed: logger.report(False, 'Suite halted on test failure') logger.testsSkipped = len(tests.split(',')) - (tests.split(',').index(paramSet) + 1) break if logger.debug < 2: checkRepo(logger) logger.endSuite() if logger.debug == 0: logger.easyReadSummary() else: logger.summary() logger.simpleSummary() logger.tinderOutput() scripting.app_ns().root.Quit()
def __init__(self, name, logger, recurrence=1, appendVar='', printAppend='', appendDict={}, appendList=[]): """ Instantiation method. Required Arguments: name: str -- Name of test. logger: TestOutput -- Instance of TestOuput object. Keyword Arguments: recurrence: int -- Amount of times you wish to run self.recurringTest method in test classes, used by TestObject appendDict: dict -- Used by test class writers when executing tests and writing self.startTest methods. Currently not needed unless you run test classes in threaded stress testing env. appendList: list -- List used for appends by test class writers. appendVar: str -- String used for conveinient test class appends. printAppend: str -- Used by TestObject.printOut method for appending to print. Useful when appending thread number in stress testing framework. """ self.results = [] self.resultNames = [] self.resultComments = [] self.recurrence = recurrence self.appendVar = str(appendVar) self.printAppend = printAppend self.appendDict = appendDict self.appendList = appendList self.logger = logger self.name = name self.scripting = scripting self.app_ns = scripting.app_ns() # initialize preferences pim_ns = schema.ns('osaf.pim', self.app_ns.itsView) pim_ns.TimezonePrefs.showPrompt = False sys.stderr = stderr_replacement(self.logger)
def Close(self, quit=True): now = datetime.now() if self.toClose: # The file must close (time to report a summary) TestLogger.logger = None elapsed = now - self.startDate elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0 self.Print("") self.Print( "++++++++++++++++++++++++SUMMARY++++++++++++++++++++++++") self.Print("Start date (before %s test script) = %s" % (self.mainDescription, self.startDate)) self.Print("End date (after %s test script) = %s" % (self.mainDescription, now)) self.Print("Time Elapsed = %s seconds" % elapsed_secs) self.Print("") #display the TestSuite status summary if not len(self.testcaseList) == 0: self.Print("TestSuite Summary :") nbTCFailed = 0 nbTCPassed = 0 nbTCUnchecked = 0 for tc in self.testcaseList: if tc[1] == "FAILED": nbTCFailed = nbTCFailed + 1 elif tc[1] == "PASSED": nbTCPassed = nbTCPassed + 1 else: nbTCUnchecked = nbTCUnchecked + 1 self.Print("- %s = %s" % (tc[0], tc[1])) self.Print("") self.Print("Total number of Testcases RUN : %s" % len(self.testcaseList)) self.Print("Total number of Testcases PASSED : %s" % nbTCPassed) self.Print("Total number of Testcases FAILED : %s" % nbTCFailed) self.Print("") # compute the status of the Testscript if is composed by sub-testcases if nbTCPassed == len(self.testcaseList): status = "PASSED" else: status = "FAILED" # Test suite status self.Print("Status : %s test suite %s" % (self.mainDescription, status)) # compute the status of the main testcase if is composed by actions else: if self.nbUnchecked == self.nbVerif: status = "UNCHECKED" elif self.nbPass == self.nbVerif: status = "PASSED" else: status = "FAILED" self.Print("") #Test case status self.Print("Status : %s testcase %s" % (self.mainDescription, status)) #change to fail if output detected on stderr if self.standardErr.hadError: status = "FAILED" self.Print(self.standardErr.text) # Tinderbox printing # compute the elapsed time in seconds elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0 description = string.join(string.split(self.mainDescription, " "), "_") self.Print("") self.Print("#TINDERBOX# Testname = %s" % description) self.Print("#TINDERBOX# Status = %s" % status) self.Print("#TINDERBOX# Time elapsed = %s (seconds)" % elapsed_secs) if status == "PASSED": self.PrintTBOX(elapsed) else: wx.GetApp().exitValue = 1 self.Print("") self.Print( "******* End of Report *******") print("#TINDERBOX# Testname = %s" % description) print("#TINDERBOX# Status = %s" % status) print("#TINDERBOX# Time elapsed = %s (seconds)" % elapsed_secs) #print names of failed tests and stderr output if there was any if status == "FAILED": for tc in self.testcaseList: if tc[1] == "FAILED": self.PrintBoth(tc[0] + ' failed') if not self.inTerminal: # close the file self.File.close() # quit Chandler if quit: import osaf.framework.scripting as scripting scripting.app_ns().root.Quit() else: # Just the end of a testcase if self.subTestcaseDesc: if self.nbUnchecked == self.nbVerif: status = "UNCHECKED" elif self.nbPass == self.nbVerif: status = "PASSED" else: status = "FAILED" self.testcaseList.append((self.subTestcaseDesc, status)) # Test case status elapsed = now - self.testcaseStartDate elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0 self.Print("----- %s summary -----" % self.subTestcaseDesc) self.Print("Start date (before %s testcase) = %s" % (self.subTestcaseDesc, self.testcaseStartDate)) self.Print("End date (after %s testcase) = %s" % (self.subTestcaseDesc, now)) self.Print("Time Elapsed = %s seconds" % elapsed_secs) self.Print("Testcase Name= %s" % self.subTestcaseDesc) self.Print("Testcase Status = %s" % status) if status == "PASSED": self.PrintTBOX(elapsed, "Testcase") else: wx.GetApp().exitValue = 1 # new testcase inits self.Reset()
def Close(self, quit=True): now = datetime.now() if self.toClose: # The file must close (time to report a summary) TestLogger.logger = None elapsed = now - self.startDate elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0 self.Print("") self.Print("++++++++++++++++++++++++SUMMARY++++++++++++++++++++++++") self.Print("Start date (before %s test script) = %s" %(self.mainDescription, self.startDate)) self.Print("End date (after %s test script) = %s" %(self.mainDescription, now)) self.Print("Time Elapsed = %s seconds" % elapsed_secs) self.Print("") #display the TestSuite status summary if not len(self.testcaseList) == 0: self.Print("TestSuite Summary :") nbTCFailed = 0 nbTCPassed = 0 nbTCUnchecked = 0 for tc in self.testcaseList: if tc[1] == "FAILED": nbTCFailed = nbTCFailed + 1 elif tc[1] == "PASSED": nbTCPassed = nbTCPassed + 1 else: nbTCUnchecked = nbTCUnchecked + 1 self.Print("- %s = %s" %(tc[0],tc[1])) self.Print("") self.Print("Total number of Testcases RUN : %s" %len(self.testcaseList)) self.Print("Total number of Testcases PASSED : %s" %nbTCPassed) self.Print("Total number of Testcases FAILED : %s" %nbTCFailed) self.Print("") # compute the status of the Testscript if is composed by sub-testcases if nbTCPassed == len(self.testcaseList): status = "PASSED" else: status = "FAILED" # Test suite status self.Print("Status : %s test suite %s" %(self.mainDescription, status)) # compute the status of the main testcase if is composed by actions else: if self.nbUnchecked == self.nbVerif: status = "UNCHECKED" elif self.nbPass == self.nbVerif: status = "PASSED" else: status = "FAILED" self.Print("") #Test case status self.Print("Status : %s testcase %s" %(self.mainDescription, status)) #change to fail if output detected on stderr if self.standardErr.hadError: status = "FAILED" self.Print(self.standardErr.text) # Tinderbox printing # compute the elapsed time in seconds elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0 description = string.join(string.split(self.mainDescription, " "), "_") self.Print("") self.Print("#TINDERBOX# Testname = %s" %description) self.Print("#TINDERBOX# Status = %s" %status) self.Print("#TINDERBOX# Time elapsed = %s (seconds)" %elapsed_secs) if status == "PASSED": self.PrintTBOX(elapsed) else: wx.GetApp().exitValue = 1 self.Print("") self.Print("******* End of Report *******") print("#TINDERBOX# Testname = %s" %description) print("#TINDERBOX# Status = %s" %status) print("#TINDERBOX# Time elapsed = %s (seconds)" %elapsed_secs) #print names of failed tests and stderr output if there was any if status == "FAILED": for tc in self.testcaseList: if tc[1] == "FAILED": self.PrintBoth( tc[0] + ' failed') if not self.inTerminal: # close the file self.File.close() # quit Chandler if quit: import osaf.framework.scripting as scripting scripting.app_ns().root.Quit() else: # Just the end of a testcase if self.subTestcaseDesc: if self.nbUnchecked == self.nbVerif: status = "UNCHECKED" elif self.nbPass == self.nbVerif: status = "PASSED" else: status = "FAILED" self.testcaseList.append((self.subTestcaseDesc,status)) # Test case status elapsed = now - self.testcaseStartDate elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0 self.Print("----- %s summary -----" %self.subTestcaseDesc) self.Print("Start date (before %s testcase) = %s" %(self.subTestcaseDesc, self.testcaseStartDate)) self.Print("End date (after %s testcase) = %s" %(self.subTestcaseDesc, now)) self.Print("Time Elapsed = %s seconds" % elapsed_secs) self.Print("Testcase Name= %s" %self.subTestcaseDesc) self.Print("Testcase Status = %s" %status) if status == "PASSED": self.PrintTBOX(elapsed, "Testcase") else: wx.GetApp().exitValue = 1 # new testcase inits self.Reset()