"""This is a work-in-progress example to show modification of an existing
set of suites to check if any Reference tests have changed output results
significantly, and report on this.
Example created:- 2011/04/20, PatrickSunter"""
import os, sys
import credo.jobrunner
import credo.systest.systestrunner as srRunner
# This is a factor increase from the existing defined error tolerance
# (which is specified in the original test suite)
allowedTolChange = 1.1
suiteModNames = ["SysTest.RegressionTests.testAll_lowres"]
tSuites = srRunner.getSuitesFromModules(suiteModNames)
# TODO: really should post-proc from existing, but this isn't working perfectly
# yet for basic sys tests.
problemTests = []
jobRunner = credo.jobrunner.defaultRunner()
for tSuite in tSuites:
for sysTest in tSuite.sysTests:
if sysTest.testType == "Reference":
sysTest.setupTest()
testResult, mResults = sysTest.runTest(jobRunner,
postProcFromExisting=False, createReports=True)
for tCompDict in sysTest.testComps:
for tcName, tComp in tCompDict.items():
fErrors = tComp.fieldErrors
for fName, error in fErrors.items():
#!/usr/bin/env python
"""This is a work-in-progress example to show modification of an existing
set of suites to check if any Reference tests have changed output results
significantly, and report on this.
Example created:- 2011/04/20, PatrickSunter"""
import os, sys
import credo.jobrunner
import credo.systest.systestrunner as srRunner
# This is a factor increase from the existing defined error tolerance
# (which is specified in the original test suite)
allowedTolChange = 1.1
suiteModNames = ["SysTest.RegressionTests.testAll_lowres"]
tSuites = srRunner.getSuitesFromModules(suiteModNames)
# TODO: really should post-proc from existing, but this isn't working perfectly
# yet for basic sys tests.
problemTests = []
jobRunner = credo.jobrunner.defaultRunner()
for tSuite in tSuites:
for sysTest in tSuite.sysTests:
if sysTest.testType == "Reference":
sysTest.setupTest()
testResult, mResults = sysTest.runTest(jobRunner,
postProcFromExisting=False,
createReports=True)
for tCompDict in sysTest.testComps:
for tcName, tComp in tCompDict.items():
fErrors = tComp.fieldErrors
#!/usr/bin/env python
"""A small utility script to post-process all of the CREDO benchmark suites
in the current directory (matching the wildcard pattern credo_*.pdf)."""
import glob, fnmatch
import credo.systest
from credo.systest.systestrunner import getSuitesFromModules
credoFiles = glob.glob("credo_*.py")
credoFiles.sort()
credoModNames = [cf.rstrip(".py") for cf in credoFiles]
tSuites = getSuitesFromModules(credoModNames)
sRunner = credo.systest.SysTestRunner()
sRunner.runSuites(tSuites, postProcFromExisting=True, createReports=True)
print "PDF Reports generated are:"
for ts in tSuites:
pdfReps = []
for sysTest in ts.sysTests:
reps = fnmatch.filter(sysTest.generatedReports, '*.pdf')
pdfReps.extend(reps)
for rep in pdfReps:
print "%s" % rep
