def readFromStgXML(self, inputFilesList, basePath):
'''Read in the list of fields that have already been specified to
be tested from a set of StGermain input files. Useful when e.g.
working with an Analytic Solution plugin.'''
self.fromXML = True
# create a flattened file
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
inputFilesList, basePath)
ffile = stgxml.createFlattenedXML(absInputFiles)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
# Go and grab necessary info from XML file
componentEl = stgxml.getStructNode(stgRoot, self.stgXMLCompListName)
fieldTestDataEl = stgxml.getStructNode(componentEl,
self.stgXMLCompName)
fieldTestListEl = stgxml.getListNode(fieldTestDataEl,
self.stgXMLSpecFList)
fieldTestStructEls = fieldTestListEl.getchildren()
for fieldStruct in fieldTestStructEls:
numericField = stgxml.getParamValue(fieldStruct, 'NumericField',
str)
self.fields[numericField] = FieldComparisonOp(numericField)
# NB: not reading in all the other specifying stuff currently. Possibly
# would be useful to do this in future.
os.remove(ffile)
def readFromStgXML(self, inputFilesList, basePath):
'''Read in the list of fields that have already been specified to
be tested from a set of StGermain input files. Useful when e.g.
working with an Analytic Solution plugin.'''
self.fromXML = True
# create a flattened file
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(inputFilesList,
basePath)
ffile=stgxml.createFlattenedXML(absInputFiles)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
# Go and grab necessary info from XML file
componentEl = stgxml.getStructNode(stgRoot, self.stgXMLCompListName)
fieldTestDataEl = stgxml.getStructNode(componentEl, self.stgXMLCompName)
fieldTestListEl = stgxml.getListNode(fieldTestDataEl, self.stgXMLSpecFList)
fieldTestStructEls = fieldTestListEl.getchildren()
for fieldStruct in fieldTestStructEls:
numericField=stgxml.getParamValue(fieldStruct,'NumericField',str)
self.fields[numericField]=FieldComparisonOp(numericField)
# NB: not reading in all the other specifying stuff currently. Possibly
# would be useful to do this in future.
os.remove(ffile)
def readFromStgXML(self, inputFilesList, basePath):
'''Read in the list of fields that have already been specified to
be tested from a set of StGermain input files. Useful when e.g.
working with an Analytic Solution plugin.'''
self.fromXML = True
# create a flattened file
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
inputFilesList, basePath)
ffile = stgxml.createFlattenedXML(absInputFiles)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
# Go and grab necessary info from XML file
fieldTestDataEl = stgxml.getStructNode(stgRoot, self.stgXMLSpecName)
fieldTestListEl = stgxml.getListNode(fieldTestDataEl,
self.stgXMLSpecFList)
fieldTestEls = fieldTestListEl.getchildren()
# As per the current spec, the field names are followed by an index
# of analytic solution
ii = 0
while ii < len(fieldTestEls):
fieldName = fieldTestEls[ii].text
self.fields[fieldName] = FieldComparisonOp(fieldName)
# Skip the index
ii += 1
ii += 1
# NB: not reading in all the other specifying stuff currently. Possibly
# would be useful to do this in future.
os.remove(ffile)
def __init__(self,
testType,
inputFiles,
outputPathBase,
basePath=None,
nproc=1,
timeout=None,
paramOverrides=None,
solverOpts=None,
nameSuffix=None):
if isinstance(inputFiles, str):
inputFiles = [inputFiles]
testName = getStdTestName(testType + "Test", inputFiles, nproc,
paramOverrides, solverOpts, nameSuffix)
if basePath is None:
# Since this is a virtual class, get calling path 2 levels up
basePath = credo.utils.getCallingPath(2)
SysTest.__init__(self, testType, testName, basePath, outputPathBase,
nproc, timeout)
self.inputFiles = inputFiles
self.paramOverrides = paramOverrides
if self.paramOverrides == None:
self.paramOverrides = {}
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
self.inputFiles, self.basePath)
stgpath.checkAllXMLInputFilesExist(absInputFiles)
self.solverOpts = solverOpts
def test_convertLocalXMLFilesToAbsPaths_ConvertNeeded(self):
testCallingPath = "Underworld/SysTests"
sampleXMLsOrig = glob.glob(
os.path.join("sampleData", "stgXML", "*.xml"))
sampleXMLsOrig.append(os.path.join("rel", "notExist.xml"))
sampleXMLsUp = stgpath.convertLocalXMLFilesToAbsPaths(
sampleXMLsOrig, testCallingPath)
self.assertEqual(sampleXMLsOrig[:-1], sampleXMLsUp[:-1])
self.assertEqual(sampleXMLsUp[-1],
os.path.join(testCallingPath, "rel", "notExist.xml"))
def test_convertLocalXMLFilesToAbsPaths_ConvertNeeded(self):
testCallingPath = "Underworld/SysTests"
sampleXMLsOrig = glob.glob(os.path.join(
"sampleData", "stgXML", "*.xml"))
sampleXMLsOrig.append(os.path.join("rel","notExist.xml"))
sampleXMLsUp = stgpath.convertLocalXMLFilesToAbsPaths(sampleXMLsOrig,
testCallingPath)
self.assertEqual(sampleXMLsOrig[:-1], sampleXMLsUp[:-1])
self.assertEqual(sampleXMLsUp[-1],
os.path.join(testCallingPath, "rel", "notExist.xml"))
def checkStgXMLResultsEnabled(self, inputFilesList, basePath):
"""Checks that the field comparison has the writing of comparison
info to file enabled (returning Bool)."""
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
inputFilesList, basePath)
ffile = stgxml.createFlattenedXML(absInputFiles)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
fieldTestDataEl = stgxml.getStructNode(stgRoot, self.stgXMLSpecName)
appendNode = stgxml.getParamNode(fieldTestDataEl,
"appendToAnalysisFile")
appendBool = stgxml.strToBool(appendNode.text)
os.remove(ffile)
return appendBool
def checkStgXMLResultsEnabled(self, inputFilesList, basePath):
"""Checks that the field comparison has the writing of comparison
info to file enabled (returning Bool)."""
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(inputFilesList,
basePath)
ffile=stgxml.createFlattenedXML(absInputFiles)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
fieldTestDataEl = stgxml.getStructNode(stgRoot, self.stgXMLSpecName)
appendNode = stgxml.getParamNode(fieldTestDataEl,
"appendToAnalysisFile")
appendBool = stgxml.strToBool(appendNode.text)
os.remove(ffile)
return appendBool
def readFromStgXML(self, inputFilesList, basePath, cmdLineOverrides):
'''Reads all the parameters of this class from a given StGermain
set of input files'''
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
inputFilesList, basePath)
ffile = stgxml.createFlattenedXML(absInputFiles, cmdLineOverrides)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
for param, stgParam in self.stgParamInfos.iteritems():
# some of these may be none, but is ok since will check below
val = stgxml.getParamValue(stgRoot, stgParam.stgName,\
stgParam.pType)
self.setParam(param, val)
self.checkValidParams()
os.remove(ffile)
def readFromStgXML(self, inputFilesList, basePath, cmdLineOverrides):
'''Reads all the parameters of this class from a given StGermain
set of input files'''
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
inputFilesList, basePath)
ffile=stgxml.createFlattenedXML(absInputFiles, cmdLineOverrides)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
for param, stgParam in self.stgParamInfos.iteritems():
# some of these may be none, but is ok since will check below
val = stgxml.getParamValue(stgRoot, stgParam.stgName,\
stgParam.pType)
self.setParam(param, val)
self.checkValidParams()
os.remove(ffile)
def addStdTest(self, testClass, inputFiles, **testOpts):
"""Instantiate and add a "standard" system test type to the list
of System tests to be run. (The "standard" refers to the user needing
to have access to the module containing the system test type to be
added, usually from a `from credo.systest import *` statement.
:param testClass: Python class of the System test to be added. This
needs to be a sub-class of :class:`~credo.systest.api.SysTest`.
:param inputFiles: model input files to be passed through to the
System test when instantiated.
:param `**testOpts`: any other keyword arguments the user wishes to
passed through to the System test when it's instantiated.
Can be used to customise a test.
:returns: a reference to the newly created and added SysTest."""
if not inspect.isclass(testClass):
raise TypeError("The testClass argument must be a type that's"\
" a subclass of the CREDO SysTest type. Arg passed in, '%s',"\
" of type '%s', is not a Python Class." \
% (testClass, type(testClass)))
if not issubclass(testClass, testapi.SingleModelSysTest):
raise TypeError("The testClass argument must be a type that's"\
" a subclass of the CREDO SysTest type. Type passed in, '%s',"\
" not a subclass of SysTest." \
% (testClass))
callingPath = credo.utils.getCallingPath(1)
# If just given a single input file as a string, convert
# to a list (containing that single file).
if isinstance(inputFiles, str):
inputFiles = [inputFiles]
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
inputFiles, callingPath)
stgpath.checkAllXMLInputFilesExist(absInputFiles)
if 'nproc' not in testOpts:
testOpts['nproc'] = self.nproc
outputPath = testapi.getStdOutputPath(testClass, inputFiles, testOpts)
newSysTest = testClass(inputFiles,
outputPath,
basePath=callingPath,
**testOpts)
self.sysTests.append(newSysTest)
return newSysTest
def addStdTest(self, testClass, inputFiles, **testOpts):
"""Instantiate and add a "standard" system test type to the list
of System tests to be run. (The "standard" refers to the user needing
to have access to the module containing the system test type to be
added, usually from a `from credo.systest import *` statement.
:param testClass: Python class of the System test to be added. This
needs to be a sub-class of :class:`~credo.systest.api.SysTest`.
:param inputFiles: model input files to be passed through to the
System test when instantiated.
:param `**testOpts`: any other keyword arguments the user wishes to
passed through to the System test when it's instantiated.
Can be used to customise a test.
:returns: a reference to the newly created and added SysTest."""
if not inspect.isclass(testClass):
raise TypeError("The testClass argument must be a type that's"\
" a subclass of the CREDO SysTest type. Arg passed in, '%s',"\
" of type '%s', is not a Python Class." \
% (testClass, type(testClass)))
if not issubclass(testClass, testapi.SingleModelSysTest):
raise TypeError("The testClass argument must be a type that's"\
" a subclass of the CREDO SysTest type. Type passed in, '%s',"\
" not a subclass of SysTest." \
% (testClass))
callingPath = credo.utils.getCallingPath(1)
# If just given a single input file as a string, convert
# to a list (containing that single file).
if isinstance(inputFiles, str):
inputFiles = [inputFiles]
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(inputFiles,
callingPath)
stgpath.checkAllXMLInputFilesExist(absInputFiles)
if 'nproc' not in testOpts:
testOpts['nproc']=self.nproc
outputPath = testapi.getStdOutputPath(testClass, inputFiles, testOpts)
newSysTest = testClass(inputFiles, outputPath, basePath=callingPath,
**testOpts)
self.sysTests.append(newSysTest)
return newSysTest
def getNumEls(mRun):
"""Calculate the number of elements used by a model run."""
#TODO: this should be abstracted a service of models perhaps ...
#res = mRun.getResolution()
from credo.io import stgxml
from credo.io import stgpath
import credo.modelrun as mrun
from xml.etree import ElementTree as etree
paramOverridesStr = mrun.getParamOverridesAsStr(mRun.paramOverrides)
iFiles = mRun.modelInputFiles
if mRun.analysisXML is not None: iFiles += [mRun.analysisXML]
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
iFiles, mRun.basePath)
ffile = stgxml.createFlattenedXML(absInputFiles, paramOverridesStr)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
resX = stgxml.getParamValue(stgRoot, "elementResI", int)
resY = stgxml.getParamValue(stgRoot, "elementResJ", int)
resZ = stgxml.getParamValue(stgRoot, "elementResK", int)
if resZ == 0: resZ = 1 #(Just in case)
numEls = resX * resY * resZ
return numEls
def getNumEls(mRun):
"""Calculate the number of elements used by a model run."""
#TODO: this should be abstracted a service of models perhaps ...
#res = mRun.getResolution()
from credo.io import stgxml
from credo.io import stgpath
import credo.modelrun as mrun
from xml.etree import ElementTree as etree
paramOverridesStr = mrun.getParamOverridesAsStr(mRun.paramOverrides)
iFiles = mRun.modelInputFiles
if mRun.analysisXML is not None: iFiles += [mRun.analysisXML]
absInputFiles = stgpath.convertLocalXMLFilesToAbsPaths(
iFiles, mRun.basePath)
ffile = stgxml.createFlattenedXML(absInputFiles, paramOverridesStr)
xmlDoc = etree.parse(ffile)
stgRoot = xmlDoc.getroot()
resX = stgxml.getParamValue(stgRoot, "elementResI", int)
resY = stgxml.getParamValue(stgRoot, "elementResJ", int)
resZ = stgxml.getParamValue(stgRoot, "elementResK", int)
if resZ == 0: resZ = 1 #(Just in case)
numEls = resX * resY * resZ
return numEls
def test_convertLocalXMLFilesToAbsPaths(self):
sampleXMLsOrig = glob.glob(
os.path.join("sampleData", "stgXML", "*.xml"))
sampleXMLsUp = stgpath.convertLocalXMLFilesToAbsPaths(
sampleXMLsOrig, "Underworld/SysTests")
self.assertEqual(sampleXMLsOrig, sampleXMLsUp)
def test_convertLocalXMLFilesToAbsPaths(self):
sampleXMLsOrig = glob.glob(os.path.join(
"sampleData", "stgXML", "*.xml"))
sampleXMLsUp = stgpath.convertLocalXMLFilesToAbsPaths(sampleXMLsOrig,
"Underworld/SysTests")
self.assertEqual(sampleXMLsOrig, sampleXMLsUp)