def run_perf_tests(tests, debug=testDebug, mask=testMask, logName=logFileName):
"""Method to execute cats tests, must be in Performance directory"""
logger = TestOutput(stdout=True, debug=0, logName=logName)
logger.startSuite(name='ChandlerTestSuite')
for paramSet in tests.split(','):
try:
filenameAndTest = paramSet.split(':')
#assume file name and and test name are the same if only one given
if len(filenameAndTest) < 2:
filenameAndTest.append(filenameAndTest[0])
teststring = 'from tools.cats.Performance.%s import %s' % (
filenameAndTest[0], filenameAndTest[1])
exec(compile(teststring, '', 'exec'))
teststring = 'test = %s(name=\'%s\', logger=logger)' % (
filenameAndTest[1], filenameAndTest[1])
exec(compile(teststring, '', 'exec'))
test.runTest()
if logger.debug == 2: checkRepo(logger)
except:
logger.traceback()
logger.endSuite()
if logger.debug == 0:
logger.easyReadSummary()
else:
logger.summary()
logger.simpleSummary()
logger.tinderOutput()
if logger.debug < 2: checkRepo(logger)
logger.endSuite()
import osaf.framework.scripting as scripting
scripting.app_ns().root.Quit()
def run_perf_tests(tests, debug=testDebug, mask=testMask, logName=logFileName):
"""Method to execute cats tests, must be in Performance directory"""
logger = TestOutput(stdout=True, debug=0, logName=logName)
logger.startSuite(name='ChandlerTestSuite')
for paramSet in tests.split(','):
try:
filenameAndTest = paramSet.split(':')
#assume file name and and test name are the same if only one given
if len(filenameAndTest) < 2: filenameAndTest.append(filenameAndTest[0])
teststring = 'from tools.cats.Performance.%s import %s' % (filenameAndTest[0], filenameAndTest[1])
exec(compile(teststring, '', 'exec'))
teststring = 'test = %s(name=\'%s\', logger=logger)' % (filenameAndTest[1], filenameAndTest[1])
exec(compile(teststring, '', 'exec'))
test.runTest()
if logger.debug == 2: checkRepo(logger)
except:
logger.traceback()
logger.endSuite()
if logger.debug == 0:
logger.easyReadSummary()
else:
logger.summary()
logger.simpleSummary()
logger.tinderOutput()
if logger.debug < 2: checkRepo(logger)
logger.endSuite()
import osaf.framework.scripting as scripting
scripting.app_ns().root.Quit()
def run_tests(tests, debug=testDebug, mask=testMask, logName=logFileName):
"""Method to execute cats tests, must be in Functional directory."""
logger = TestOutput(stdout=True, debug=debug, mask=mask, logName=logName)
testList = tests.split(',')
statusBar = Block.findBlockByName("StatusBar")
totalTests = len(tests.split(','))
curTest = 0
if len(testList) < 2:
logger.startSuite(name=testList[0].split(':')[0])
else:
logger.startSuite(name='ChandlerFunctionalTestSuite')
# We'll normally run individual tests with an exception wrapper;
# --catch=never will turn it off, so that a debugger can stop on
# uncaught exceptions.
runner = Globals.options.catch != 'never' and run_test_wrapped or run_test
for paramSet in testList:
curTest += 1
statusBar.setStatusMessage(
"Test %d of %d %s" % (curTest, totalTests, paramSet.split(":")[0]))
runner(logger, paramSet)
if haltOnFailure and logger.testHasFailed:
logger.report(False, 'Suite halted on test failure')
logger.testsSkipped = len(
tests.split(',')) - (tests.split(',').index(paramSet) + 1)
break
if logger.debug < 2: checkRepo(logger)
logger.endSuite()
if logger.debug == 0:
logger.easyReadSummary()
else:
logger.summary()
logger.simpleSummary()
logger.tinderOutput()
scripting.app_ns().root.Quit()
def run_tests(tests, debug=testDebug, mask=testMask, logName=logFileName):
"""Method to execute cats tests, must be in Functional directory."""
logger = TestOutput(stdout=True, debug=debug, mask=mask, logName=logName)
testList = tests.split(',')
statusBar = Block.findBlockByName("StatusBar")
totalTests = len(tests.split(','))
curTest = 0
if len(testList) < 2:
logger.startSuite(name = testList[0].split(':')[0])
else:
logger.startSuite(name='ChandlerFunctionalTestSuite')
# We'll normally run individual tests with an exception wrapper;
# --catch=never will turn it off, so that a debugger can stop on
# uncaught exceptions.
runner = Globals.options.catch != 'never' and run_test_wrapped or run_test
for paramSet in testList:
curTest += 1
statusBar.setStatusMessage("Test %d of %d %s" % (curTest, totalTests, paramSet.split(":")[0]))
runner(logger, paramSet)
if haltOnFailure and logger.testHasFailed:
logger.report(False, 'Suite halted on test failure')
logger.testsSkipped = len(tests.split(',')) - (tests.split(',').index(paramSet) + 1)
break
if logger.debug < 2: checkRepo(logger)
logger.endSuite()
if logger.debug == 0:
logger.easyReadSummary()
else:
logger.summary()
logger.simpleSummary()
logger.tinderOutput()
scripting.app_ns().root.Quit()
def __init__(self, name, logger, recurrence=1, appendVar='', printAppend='', appendDict={}, appendList=[]):
"""
Instantiation method.
Required Arguments:
name: str -- Name of test.
logger: TestOutput -- Instance of TestOuput object.
Keyword Arguments:
recurrence: int -- Amount of times you wish to run self.recurringTest method in test classes,
used by TestObject
appendDict: dict -- Used by test class writers when executing tests and writing self.startTest methods.
Currently not needed unless you run test classes in threaded stress testing env.
appendList: list -- List used for appends by test class writers.
appendVar: str -- String used for conveinient test class appends.
printAppend: str -- Used by TestObject.printOut method for appending to print. Useful when appending
thread number in stress testing framework.
"""
self.results = []
self.resultNames = []
self.resultComments = []
self.recurrence = recurrence
self.appendVar = str(appendVar)
self.printAppend = printAppend
self.appendDict = appendDict
self.appendList = appendList
self.logger = logger
self.name = name
self.scripting = scripting
self.app_ns = scripting.app_ns()
# initialize preferences
pim_ns = schema.ns('osaf.pim', self.app_ns.itsView)
pim_ns.TimezonePrefs.showPrompt = False
sys.stderr = stderr_replacement(self.logger)
def Close(self, quit=True):
now = datetime.now()
if self.toClose: # The file must close (time to report a summary)
TestLogger.logger = None
elapsed = now - self.startDate
elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0
self.Print("")
self.Print(
"++++++++++++++++++++++++SUMMARY++++++++++++++++++++++++")
self.Print("Start date (before %s test script) = %s" %
(self.mainDescription, self.startDate))
self.Print("End date (after %s test script) = %s" %
(self.mainDescription, now))
self.Print("Time Elapsed = %s seconds" % elapsed_secs)
self.Print("")
#display the TestSuite status summary
if not len(self.testcaseList) == 0:
self.Print("TestSuite Summary :")
nbTCFailed = 0
nbTCPassed = 0
nbTCUnchecked = 0
for tc in self.testcaseList:
if tc[1] == "FAILED":
nbTCFailed = nbTCFailed + 1
elif tc[1] == "PASSED":
nbTCPassed = nbTCPassed + 1
else:
nbTCUnchecked = nbTCUnchecked + 1
self.Print("- %s = %s" % (tc[0], tc[1]))
self.Print("")
self.Print("Total number of Testcases RUN : %s" %
len(self.testcaseList))
self.Print("Total number of Testcases PASSED : %s" %
nbTCPassed)
self.Print("Total number of Testcases FAILED : %s" %
nbTCFailed)
self.Print("")
# compute the status of the Testscript if is composed by sub-testcases
if nbTCPassed == len(self.testcaseList):
status = "PASSED"
else:
status = "FAILED"
# Test suite status
self.Print("Status : %s test suite %s" %
(self.mainDescription, status))
# compute the status of the main testcase if is composed by actions
else:
if self.nbUnchecked == self.nbVerif:
status = "UNCHECKED"
elif self.nbPass == self.nbVerif:
status = "PASSED"
else:
status = "FAILED"
self.Print("")
#Test case status
self.Print("Status : %s testcase %s" %
(self.mainDescription, status))
#change to fail if output detected on stderr
if self.standardErr.hadError:
status = "FAILED"
self.Print(self.standardErr.text)
# Tinderbox printing
# compute the elapsed time in seconds
elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0
description = string.join(string.split(self.mainDescription, " "),
"_")
self.Print("")
self.Print("#TINDERBOX# Testname = %s" % description)
self.Print("#TINDERBOX# Status = %s" % status)
self.Print("#TINDERBOX# Time elapsed = %s (seconds)" %
elapsed_secs)
if status == "PASSED":
self.PrintTBOX(elapsed)
else:
wx.GetApp().exitValue = 1
self.Print("")
self.Print(
"******* End of Report *******")
print("#TINDERBOX# Testname = %s" % description)
print("#TINDERBOX# Status = %s" % status)
print("#TINDERBOX# Time elapsed = %s (seconds)" % elapsed_secs)
#print names of failed tests and stderr output if there was any
if status == "FAILED":
for tc in self.testcaseList:
if tc[1] == "FAILED":
self.PrintBoth(tc[0] + ' failed')
if not self.inTerminal:
# close the file
self.File.close()
# quit Chandler
if quit:
import osaf.framework.scripting as scripting
scripting.app_ns().root.Quit()
else: # Just the end of a testcase
if self.subTestcaseDesc:
if self.nbUnchecked == self.nbVerif:
status = "UNCHECKED"
elif self.nbPass == self.nbVerif:
status = "PASSED"
else:
status = "FAILED"
self.testcaseList.append((self.subTestcaseDesc, status))
# Test case status
elapsed = now - self.testcaseStartDate
elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0
self.Print("----- %s summary -----" % self.subTestcaseDesc)
self.Print("Start date (before %s testcase) = %s" %
(self.subTestcaseDesc, self.testcaseStartDate))
self.Print("End date (after %s testcase) = %s" %
(self.subTestcaseDesc, now))
self.Print("Time Elapsed = %s seconds" % elapsed_secs)
self.Print("Testcase Name= %s" % self.subTestcaseDesc)
self.Print("Testcase Status = %s" % status)
if status == "PASSED":
self.PrintTBOX(elapsed, "Testcase")
else:
wx.GetApp().exitValue = 1
# new testcase inits
self.Reset()
def Close(self, quit=True):
now = datetime.now()
if self.toClose: # The file must close (time to report a summary)
TestLogger.logger = None
elapsed = now - self.startDate
elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0
self.Print("")
self.Print("++++++++++++++++++++++++SUMMARY++++++++++++++++++++++++")
self.Print("Start date (before %s test script) = %s" %(self.mainDescription, self.startDate))
self.Print("End date (after %s test script) = %s" %(self.mainDescription, now))
self.Print("Time Elapsed = %s seconds" % elapsed_secs)
self.Print("")
#display the TestSuite status summary
if not len(self.testcaseList) == 0:
self.Print("TestSuite Summary :")
nbTCFailed = 0
nbTCPassed = 0
nbTCUnchecked = 0
for tc in self.testcaseList:
if tc[1] == "FAILED":
nbTCFailed = nbTCFailed + 1
elif tc[1] == "PASSED":
nbTCPassed = nbTCPassed + 1
else:
nbTCUnchecked = nbTCUnchecked + 1
self.Print("- %s = %s" %(tc[0],tc[1]))
self.Print("")
self.Print("Total number of Testcases RUN : %s" %len(self.testcaseList))
self.Print("Total number of Testcases PASSED : %s" %nbTCPassed)
self.Print("Total number of Testcases FAILED : %s" %nbTCFailed)
self.Print("")
# compute the status of the Testscript if is composed by sub-testcases
if nbTCPassed == len(self.testcaseList):
status = "PASSED"
else:
status = "FAILED"
# Test suite status
self.Print("Status : %s test suite %s" %(self.mainDescription, status))
# compute the status of the main testcase if is composed by actions
else:
if self.nbUnchecked == self.nbVerif:
status = "UNCHECKED"
elif self.nbPass == self.nbVerif:
status = "PASSED"
else:
status = "FAILED"
self.Print("")
#Test case status
self.Print("Status : %s testcase %s" %(self.mainDescription, status))
#change to fail if output detected on stderr
if self.standardErr.hadError:
status = "FAILED"
self.Print(self.standardErr.text)
# Tinderbox printing
# compute the elapsed time in seconds
elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0
description = string.join(string.split(self.mainDescription, " "), "_")
self.Print("")
self.Print("#TINDERBOX# Testname = %s" %description)
self.Print("#TINDERBOX# Status = %s" %status)
self.Print("#TINDERBOX# Time elapsed = %s (seconds)" %elapsed_secs)
if status == "PASSED":
self.PrintTBOX(elapsed)
else:
wx.GetApp().exitValue = 1
self.Print("")
self.Print("******* End of Report *******")
print("#TINDERBOX# Testname = %s" %description)
print("#TINDERBOX# Status = %s" %status)
print("#TINDERBOX# Time elapsed = %s (seconds)" %elapsed_secs)
#print names of failed tests and stderr output if there was any
if status == "FAILED":
for tc in self.testcaseList:
if tc[1] == "FAILED":
self.PrintBoth( tc[0] + ' failed')
if not self.inTerminal:
# close the file
self.File.close()
# quit Chandler
if quit:
import osaf.framework.scripting as scripting
scripting.app_ns().root.Quit()
else: # Just the end of a testcase
if self.subTestcaseDesc:
if self.nbUnchecked == self.nbVerif:
status = "UNCHECKED"
elif self.nbPass == self.nbVerif:
status = "PASSED"
else:
status = "FAILED"
self.testcaseList.append((self.subTestcaseDesc,status))
# Test case status
elapsed = now - self.testcaseStartDate
elapsed_secs = elapsed.seconds + elapsed.microseconds / 1000000.0
self.Print("----- %s summary -----" %self.subTestcaseDesc)
self.Print("Start date (before %s testcase) = %s" %(self.subTestcaseDesc, self.testcaseStartDate))
self.Print("End date (after %s testcase) = %s" %(self.subTestcaseDesc, now))
self.Print("Time Elapsed = %s seconds" % elapsed_secs)
self.Print("Testcase Name= %s" %self.subTestcaseDesc)
self.Print("Testcase Status = %s" %status)
if status == "PASSED":
self.PrintTBOX(elapsed, "Testcase")
else:
wx.GetApp().exitValue = 1
# new testcase inits
self.Reset()
