"""This is a work-in-progress example to show modification of an existing set of suites to check if any Reference tests have changed output results significantly, and report on this. Example created:- 2011/04/20, PatrickSunter""" import os, sys import credo.jobrunner import credo.systest.systestrunner as srRunner # This is a factor increase from the existing defined error tolerance # (which is specified in the original test suite) allowedTolChange = 1.1 suiteModNames = ["SysTest.RegressionTests.testAll_lowres"] tSuites = srRunner.getSuitesFromModules(suiteModNames) # TODO: really should post-proc from existing, but this isn't working perfectly # yet for basic sys tests. problemTests = [] jobRunner = credo.jobrunner.defaultRunner() for tSuite in tSuites: for sysTest in tSuite.sysTests: if sysTest.testType == "Reference": sysTest.setupTest() testResult, mResults = sysTest.runTest(jobRunner, postProcFromExisting=False, createReports=True) for tCompDict in sysTest.testComps: for tcName, tComp in tCompDict.items(): fErrors = tComp.fieldErrors for fName, error in fErrors.items():
#!/usr/bin/env python """This is a work-in-progress example to show modification of an existing set of suites to check if any Reference tests have changed output results significantly, and report on this. Example created:- 2011/04/20, PatrickSunter""" import os, sys import credo.jobrunner import credo.systest.systestrunner as srRunner # This is a factor increase from the existing defined error tolerance # (which is specified in the original test suite) allowedTolChange = 1.1 suiteModNames = ["SysTest.RegressionTests.testAll_lowres"] tSuites = srRunner.getSuitesFromModules(suiteModNames) # TODO: really should post-proc from existing, but this isn't working perfectly # yet for basic sys tests. problemTests = [] jobRunner = credo.jobrunner.defaultRunner() for tSuite in tSuites: for sysTest in tSuite.sysTests: if sysTest.testType == "Reference": sysTest.setupTest() testResult, mResults = sysTest.runTest(jobRunner, postProcFromExisting=False, createReports=True) for tCompDict in sysTest.testComps: for tcName, tComp in tCompDict.items(): fErrors = tComp.fieldErrors
#!/usr/bin/env python """A small utility script to post-process all of the CREDO benchmark suites in the current directory (matching the wildcard pattern credo_*.pdf).""" import glob, fnmatch import credo.systest from credo.systest.systestrunner import getSuitesFromModules credoFiles = glob.glob("credo_*.py") credoFiles.sort() credoModNames = [cf.rstrip(".py") for cf in credoFiles] tSuites = getSuitesFromModules(credoModNames) sRunner = credo.systest.SysTestRunner() sRunner.runSuites(tSuites, postProcFromExisting=True, createReports=True) print "PDF Reports generated are:" for ts in tSuites: pdfReps = [] for sysTest in ts.sysTests: reps = fnmatch.filter(sysTest.generatedReports, '*.pdf') pdfReps.extend(reps) for rep in pdfReps: print "%s" % rep