"""This is a work-in-progress example to show modification of an existing
set of suites to check if any Reference tests have changed output results
significantly, and report on this.

Example created:- 2011/04/20, PatrickSunter"""

import os, sys
import credo.jobrunner
import credo.systest.systestrunner as srRunner

# This is a factor increase from the existing defined error tolerance
#  (which is specified in the original test suite)
allowedTolChange = 1.1
suiteModNames = ["SysTest.RegressionTests.testAll_lowres"]
tSuites = srRunner.getSuitesFromModules(suiteModNames)

# TODO: really should post-proc from existing, but this isn't working perfectly
# yet for basic sys tests.
problemTests = []
jobRunner = credo.jobrunner.defaultRunner()
for tSuite in tSuites:
    for sysTest in tSuite.sysTests:
        if sysTest.testType == "Reference":
            sysTest.setupTest()
            testResult, mResults = sysTest.runTest(jobRunner,
                    postProcFromExisting=False, createReports=True)
            for tCompDict in sysTest.testComps:
                for tcName, tComp in tCompDict.items():
                    fErrors = tComp.fieldErrors
                    for fName, error in fErrors.items():
Example #2
0
#!/usr/bin/env python
"""This is a work-in-progress example to show modification of an existing
set of suites to check if any Reference tests have changed output results
significantly, and report on this.

Example created:- 2011/04/20, PatrickSunter"""

import os, sys
import credo.jobrunner
import credo.systest.systestrunner as srRunner

# This is a factor increase from the existing defined error tolerance
#  (which is specified in the original test suite)
allowedTolChange = 1.1
suiteModNames = ["SysTest.RegressionTests.testAll_lowres"]
tSuites = srRunner.getSuitesFromModules(suiteModNames)

# TODO: really should post-proc from existing, but this isn't working perfectly
# yet for basic sys tests.
problemTests = []
jobRunner = credo.jobrunner.defaultRunner()
for tSuite in tSuites:
    for sysTest in tSuite.sysTests:
        if sysTest.testType == "Reference":
            sysTest.setupTest()
            testResult, mResults = sysTest.runTest(jobRunner,
                                                   postProcFromExisting=False,
                                                   createReports=True)
            for tCompDict in sysTest.testComps:
                for tcName, tComp in tCompDict.items():
                    fErrors = tComp.fieldErrors
#!/usr/bin/env python
"""A small utility script to post-process all of the CREDO benchmark suites
in the current directory (matching the wildcard pattern credo_*.pdf)."""

import glob, fnmatch
import credo.systest
from credo.systest.systestrunner import getSuitesFromModules

credoFiles = glob.glob("credo_*.py")
credoFiles.sort()
credoModNames = [cf.rstrip(".py") for cf in credoFiles]
tSuites = getSuitesFromModules(credoModNames)

sRunner = credo.systest.SysTestRunner()
sRunner.runSuites(tSuites, postProcFromExisting=True, createReports=True)

print "PDF Reports generated are:"
for ts in tSuites:
    pdfReps = []
    for sysTest in ts.sysTests:
        reps = fnmatch.filter(sysTest.generatedReports, '*.pdf')
        pdfReps.extend(reps)
    for rep in pdfReps:
        print "%s" % rep