def setTimePrint(self, msg):
"""
Allows the code to toggle timestamp printing.
@ In, msg, string, the string that means true or false
@ Out, None
"""
if msg.lower() in utils.stringsThatMeanTrue():
self.callerLength = 40
self.tagLength = 30
self.printTime = True
elif msg.lower() in utils.stringsThatMeanFalse():
self.callerLength = 25
self.tagLength = 15
self.printTime = False
def _readMoreXML(self, xmlNode):
"""
Handles the reading of all the XML describing the step
Since step are not reused there will not be changes in the parameter describing the step after this reading
@ In, xmlNode, xml.etree.ElementTree.Element, XML element node that represents the portion of the input that belongs to this Step class
@ Out, None
"""
printString = 'For step of type {0:15} and name {1:15} the attribute {3:10} has been assigned to a not understandable value {2:10}'
self.raiseADebug(
'move this tests to base class when it is ready for all the classes'
)
if not set(xmlNode.attrib.keys()).issubset(set(self._knownAttribute)):
self.raiseAnError(
IOError,
'In step of type {0:15} and name {1:15} there are unknown attributes {2:100}'
.format(self.type, self.name, str(xmlNode.attrib.keys())))
if 're-seeding' in xmlNode.attrib.keys():
self.initSeed = xmlNode.attrib['re-seeding']
if self.initSeed.lower() == "continue": self.initSeed = "continue"
else:
try:
self.initSeed = int(self.initSeed)
except:
self.raiseAnError(
IOError,
printString.format(self.type, self.name, self.initSeed,
're-seeding'))
if 'sleepTime' in xmlNode.attrib.keys():
try:
self.sleepTime = float(xmlNode.attrib['sleepTime'])
except:
self.raiseAnError(
IOError,
printString.format(self.type, self.name,
xmlNode.attrib['sleepTime'],
'sleepTime'))
for child in xmlNode:
classType, classSubType = child.attrib.get(
'class'), child.attrib.get('type')
if None in [classType, classSubType]:
self.raiseAnError(
IOError, "In Step named " + self.name + ", subnode " +
child.tag + ", and body content = " + child.text +
" the attribute class and/or type has not been found!")
self.parList.append([
child.tag,
child.attrib.get('class'),
child.attrib.get('type'), child.text
])
self.pauseEndStep = False
if 'pauseAtEnd' in xmlNode.attrib.keys():
if xmlNode.attrib['pauseAtEnd'].lower(
) in utils.stringsThatMeanTrue():
self.pauseEndStep = True
elif xmlNode.attrib['pauseAtEnd'].lower(
) in utils.stringsThatMeanFalse():
self.pauseEndStep = False
else:
self.raiseAnError(
IOError,
printString.format(self.type, self.name,
xmlNode.attrib['pauseAtEnd'],
'pauseAtEnd'))
self._localInputAndChecks(xmlNode)
if None in self.parList:
self.raiseAnError(
IOError,
'A problem was found in the definition of the step ' +
str(self.name))
A type that allows True or False
"""
@classmethod
def convert(cls, value):
"""
Converts value from string to a bool.
@ In, value, string, the value to convert
@ Out, convert, bool, the converted value
"""
if value.lower() in utils.stringsThatMeanTrue():
return True
else:
return False
boolTypeList = utils.stringsThatMeanTrue() + utils.stringsThatMeanFalse()
BoolType.createClass("bool", "boolType",
boolTypeList + [elm.capitalize() for elm in boolTypeList])
#
#
#
#
class Quantity:
"""
A class that allows the quantity of a node to be specified.
If python3.4+ is required, this should be switched to a Python 3.4 Enum.
"""
zero_to_one = (0, 1)
zero_to_infinity = (0, 2)
def __readRunInfo(self,xmlNode,runInfoSkip,xmlFilename):
"""
Method that reads the xml input file for the RunInfo block
@ In, xmlNode, xml.etree.Element, the xml node that belongs to Simulation
@ In, runInfoSkip, string, the runInfo step to skip
@ In, xmlFilename, string, xml input file name
@ Out, None
"""
if 'verbosity' in xmlNode.attrib.keys():
self.verbosity = xmlNode.attrib['verbosity']
self.raiseAMessage('Global verbosity level is "',self.verbosity,'"',verbosity='quiet')
for element in xmlNode:
if element.tag in runInfoSkip:
self.raiseAWarning("Skipped element ",element.tag)
elif element.tag == 'printInput':
text = element.text.strip() if element.text is not None else ''
#extension fixing
if len(text) >= 4 and text[-4:].lower() == '.xml':
text = text[:-4]
# if the user asked to not print input instead of leaving off tag, respect it
if text.lower() in utils.stringsThatMeanFalse():
self.runInfoDict['printInput'] = False
# if the user didn't provide a name, provide a default
elif len(text)<1:
self.runInfoDict['printInput'] = 'duplicated_input.xml'
# otherwise, use the user-provided name
else:
self.runInfoDict['printInput'] = text+'.xml'
elif element.tag == 'WorkingDir':
# first store the cwd, the "CallDir"
self.runInfoDict['CallDir'] = os.getcwd()
# then get the requested "WorkingDir"
tempName = element.text
if '~' in tempName:
tempName = os.path.expanduser(tempName)
if os.path.isabs(tempName):
self.runInfoDict['WorkingDir'] = tempName
elif "runRelative" in element.attrib:
self.runInfoDict['WorkingDir'] = os.path.abspath(tempName)
else:
if xmlFilename == None:
self.raiseAnError(IOError,'Relative working directory requested but xmlFilename is None.')
# store location of the input
xmlDirectory = os.path.dirname(os.path.abspath(xmlFilename))
self.runInfoDict['InputDir'] = xmlDirectory
rawRelativeWorkingDir = element.text.strip()
# working dir is file location + relative working dir
self.runInfoDict['WorkingDir'] = os.path.join(xmlDirectory,rawRelativeWorkingDir)
utils.makeDir(self.runInfoDict['WorkingDir'])
elif element.tag == 'maxQueueSize':
try:
self.runInfoDict['maxQueueSize'] = int(element.text)
except ValueError:
self.raiseAnError('Value give for RunInfo.maxQueueSize could not be converted to integer: {}'.format(element.text))
elif element.tag == 'RemoteRunCommand':
tempName = element.text
if '~' in tempName:
tempName = os.path.expanduser(tempName)
if os.path.isabs(tempName):
self.runInfoDict['RemoteRunCommand'] = tempName
else:
self.runInfoDict['RemoteRunCommand'] = os.path.abspath(os.path.join(self.runInfoDict['FrameworkDir'],tempName))
elif element.tag == 'NodeParameter':
self.runInfoDict['NodeParameter'] = element.text.strip()
elif element.tag == 'MPIExec':
self.runInfoDict['MPIExec'] = element.text.strip()
elif element.tag == 'JobName':
self.runInfoDict['JobName' ] = element.text.strip()
elif element.tag == 'ParallelCommand':
self.runInfoDict['ParallelCommand' ] = element.text.strip()
elif element.tag == 'queueingSoftware':
self.runInfoDict['queueingSoftware' ] = element.text.strip()
elif element.tag == 'ThreadingCommand':
self.runInfoDict['ThreadingCommand' ] = element.text.strip()
elif element.tag == 'NumThreads':
self.runInfoDict['NumThreads' ] = int(element.text)
elif element.tag == 'totalNumCoresUsed':
self.runInfoDict['totalNumCoresUsed' ] = int(element.text)
elif element.tag == 'NumMPI':
self.runInfoDict['NumMPI' ] = int(element.text)
elif element.tag == 'internalParallel':
self.runInfoDict['internalParallel' ] = utils.interpretBoolean(element.text)
elif element.tag == 'batchSize':
self.runInfoDict['batchSize' ] = int(element.text)
elif element.tag.lower() == 'maxqueuesize':
self.runInfoDict['maxQueueSize' ] = int(element.text)
elif element.tag == 'MaxLogFileSize':
self.runInfoDict['MaxLogFileSize' ] = int(element.text)
elif element.tag == 'precommand':
self.runInfoDict['precommand' ] = element.text
elif element.tag == 'postcommand':
self.runInfoDict['postcommand' ] = element.text
elif element.tag == 'deleteOutExtension':
self.runInfoDict['deleteOutExtension'] = element.text.strip().split(',')
elif element.tag == 'delSucLogFiles' :
if element.text.lower() in utils.stringsThatMeanTrue():
self.runInfoDict['delSucLogFiles' ] = True
else:
self.runInfoDict['delSucLogFiles' ] = False
elif element.tag == 'logfileBuffer':
self.runInfoDict['logfileBuffer'] = utils.convertMultipleToBytes(element.text.lower())
elif element.tag == 'clusterParameters':
self.runInfoDict['clusterParameters'].extend(splitCommand(element.text)) #extend to allow adding parameters at different points.
elif element.tag == 'mode' :
self.runInfoDict['mode'] = element.text.strip().lower()
#parallel environment
if self.runInfoDict['mode'] in self.__modeHandlerDict:
self.__modeHandler = self.__modeHandlerDict[self.runInfoDict['mode']](self.messageHandler)
self.__modeHandler.XMLread(element)
else:
self.raiseAnError(IOError,"Unknown mode "+self.runInfoDict['mode'])
elif element.tag == 'expectedTime':
self.runInfoDict['expectedTime' ] = element.text.strip()
elif element.tag == 'Sequence':
for stepName in element.text.split(','):
self.stepSequenceList.append(stepName.strip())
elif element.tag == 'DefaultInputFile':
self.runInfoDict['DefaultInputFile'] = element.text.strip()
elif element.tag == 'CustomMode' :
modeName = element.text.strip()
modeClass = element.attrib["class"]
modeFile = element.attrib["file"]
#XXX This depends on if the working directory has been set yet.
# So switching the order of WorkingDir and CustomMode can
# cause different results.
modeFile = modeFile.replace("%BASE_WORKING_DIR%",self.runInfoDict['WorkingDir'])
modeFile = modeFile.replace("%FRAMEWORK_DIR%",self.runInfoDict['FrameworkDir'])
modeDir, modeFilename = os.path.split(modeFile)
if modeFilename.endswith(".py"):
modeModulename = modeFilename[:-3]
else:
modeModulename = modeFilename
os.sys.path.append(modeDir)
module = __import__(modeModulename)
if modeName in self.__modeHandlerDict:
self.raiseAWarning("duplicate mode definition " + modeName)
self.__modeHandlerDict[modeName] = module.__dict__[modeClass]
else:
self.raiseAnError(IOError,'RunInfo element "'+element.tag +'" unknown!')
def _handleInput(self, paramInput):
"""
Function to handle the parsed paramInput for this class.
@ In, paramInput, ParameterInput, the already parsed input.
@ Out, None
"""
#check if in dynamic mode; default is False
dynamicNode = paramInput.findFirst('dynamic')
if dynamicNode is not None:
#could specify as true/false or just have the node present
text = dynamicNode.value
if text is not None:
if text not in utils.stringsThatMeanFalse():
self.dynamic = True
else:
self.dynamic = True
numberOfSources = 0
for child in paramInput.subparts:
#if dynamic, accept a single file as <File ID="1" name="myOut.xml">
#if not dynamic, accept a list of files
if child.getName() == 'File':
numberOfSources += 1
if 'name' not in child.parameterValues:
self.raiseAnError(
IOError,
'Each "File" must have an associated "name"; missing for',
child.getName(), child.value)
#make sure you provide an ID and a file name
if 'ID' not in child.parameterValues:
id = 0
while id in self.files.keys():
id += 1
self.raiseAWarning(
IOError,
'Each "File" entry must have an associated "ID"; missing for',
child.getName(), child.parameterValues['name'],
'so ID is set to', id)
else:
#assure ID is a number, since it's going into a data object
id = child.parameterValues['ID']
#if already used, raise an error
if id in self.files.keys():
self.raiseAnError(
IOError, 'Multiple File nodes have the same ID:',
id)
#store id,filename pair
self.files[id] = {
'name': child.parameterValues['name'].strip(),
'fileObject': None,
'paths': {}
}
#user provides loading information as <output name="variablename">ans|pearson|x</output>
for cchild in child.subparts:
if cchild.getName() == 'output':
#make sure you provide a label for this data array
if 'name' not in cchild.parameterValues:
self.raiseAnError(
IOError,
'Must specify a "name" for each "output" block! Missing for:',
cchild.text)
varName = cchild.parameterValues['name'].strip()
if varName in self.files[id]['paths'].keys():
self.raiseAnError(
IOError,
'Multiple "output" blocks for "%s" have the same "name":'
% self.files[id]['name'], varName)
self.files[id]['paths'][varName] = cchild.value.strip()
#if dynamic, only one File can be specified currently; to fix this, how do you handle different-lengthed times in same data object?
if self.dynamic and numberOfSources > 1:
self.raiseAnError(
IOError,
'For Dynamic reading, only one "File" node can be specified! Got',
numberOfSources, 'nodes.')
# check there are entries for each
if len(self.files) < 1:
self.raiseAWarning(
'No files were specified to read from! Nothing will be done...'
)
# if no outputs listed, remove file from list and warn
toRemove = []
for id, fileDict in self.files.items():
if len(fileDict['paths']) < 1:
self.raiseAWarning(
'No outputs were specified for File with ID "%s"! No extraction will be performed for this file...'
% str(id))
toRemove.append(id)
for rem in toRemove:
del self.files[id]
def _readMoreXML(self,xmlNode):
"""
Function to read the portion of the xml input that belongs to this specialized class
and initialize some stuff based on the inputs got
@ In, xmlNode, xml.etree.ElementTree.Element, Xml element node
@ Out, None
"""
Model._readMoreXML(self, xmlNode)
paramInput = Code.getInputSpecification()()
paramInput.parseNode(xmlNode)
self.clargs={'text':'', 'input':{'noarg':[]}, 'pre':'', 'post':''} #output:''
self.fargs={'input':{}, 'output':'', 'moosevpp':''}
for child in paramInput.subparts:
if child.getName() =='executable':
self.executable = child.value if child.value is not None else ''
if child.getName() =='walltime':
self.maxWallTime = child.value
if child.getName() =='preexec':
self.preExec = child.value
elif child.getName() == 'clargs':
argtype = child.parameterValues['type'] if 'type' in child.parameterValues else None
arg = child.parameterValues['arg'] if 'arg' in child.parameterValues else None
ext = child.parameterValues['extension'] if 'extension' in child.parameterValues else None
# The default delimiter is one empty space
delimiter = child.parameterValues['delimiter'] if 'delimiter' in child.parameterValues else ' '
if argtype == None:
self.raiseAnError(IOError,'"type" for clarg not specified!')
elif argtype == 'text':
if ext != None:
self.raiseAWarning('"text" nodes only accept "type" and "arg" attributes! Ignoring "extension"...')
if not delimiter.strip():
self.raiseAWarning('"text" nodes only accept "type" and "arg" attributes! Ignoring "delimiter"...')
if arg == None:
self.raiseAnError(IOError,'"arg" for clarg '+argtype+' not specified! Enter text to be used.')
self.clargs['text']=arg
elif argtype == 'input':
if ext == None:
self.raiseAnError(IOError,'"extension" for clarg '+argtype+' not specified! Enter filetype to be listed for this flag.')
if arg == None:
self.clargs['input']['noarg'].append((ext,delimiter))
else:
if arg not in self.clargs['input'].keys():
self.clargs['input'][arg]=[]
# The delimiter is used to link 'arg' with the input file that have the file extension
# given by 'extension'. In general, empty space is used. But in some specific cases, the codes may require
# some specific delimiters to link the 'arg' and input files
self.clargs['input'][arg].append((ext,delimiter))
elif argtype == 'output':
if arg == None:
self.raiseAnError(IOError,'"arg" for clarg '+argtype+' not specified! Enter flag for output file specification.')
self.clargs['output'] = arg
elif argtype == 'prepend':
if ext != None:
self.raiseAWarning('"prepend" nodes only accept "type" and "arg" attributes! Ignoring "extension"...')
if arg == None:
self.raiseAnError(IOError,'"arg" for clarg '+argtype+' not specified! Enter text to be used.')
self.clargs['pre'] = arg
elif argtype == 'postpend':
if ext != None:
self.raiseAWarning('"postpend" nodes only accept "type" and "arg" attributes! Ignoring "extension"...')
if arg == None:
self.raiseAnError(IOError,'"arg" for clarg '+argtype+' not specified! Enter text to be used.')
self.clargs['post'] = arg
else:
self.raiseAnError(IOError,'clarg type '+argtype+' not recognized!')
elif child.getName() == 'fileargs':
argtype = child.parameterValues['type'] if 'type' in child.parameterValues else None
arg = child.parameterValues['arg'] if 'arg' in child.parameterValues else None
ext = child.parameterValues['extension'] if 'extension' in child.parameterValues else None
if argtype == None:
self.raiseAnError(IOError,'"type" for filearg not specified!')
elif argtype == 'input':
if arg == None:
self.raiseAnError(IOError,'filearg type "input" requires the template variable be specified in "arg" attribute!')
if ext == None:
self.raiseAnError(IOError,'filearg type "input" requires the auxiliary file extension be specified in "ext" attribute!')
self.fargs['input'][arg]=[ext]
elif argtype == 'output':
if self.fargs['output']!='':
self.raiseAnError(IOError,'output fileargs already specified! You can only specify one output fileargs node.')
if arg == None:
self.raiseAnError(IOError,'filearg type "output" requires the template variable be specified in "arg" attribute!')
self.fargs['output']=arg
elif argtype.lower() == 'moosevpp':
if self.fargs['moosevpp'] != '':
self.raiseAnError(IOError,'moosevpp fileargs already specified! You can only specify one moosevpp fileargs node.')
if arg == None:
self.raiseAnError(IOError,'filearg type "moosevpp" requires the template variable be specified in "arg" attribute!')
self.fargs['moosevpp']=arg
else:
self.raiseAnError(IOError,'filearg type '+argtype+' not recognized!')
if self.executable == '':
self.raiseAWarning('The node "<executable>" was not found in the body of the code model '+str(self.name)+' so no code will be run...')
else:
if os.environ.get('RAVENinterfaceCheck','False').lower() in utils.stringsThatMeanFalse():
if '~' in self.executable:
self.executable = os.path.expanduser(self.executable)
abspath = os.path.abspath(str(self.executable))
if os.path.exists(abspath):
self.executable = abspath
else:
self.raiseAMessage('not found executable '+self.executable,'ExceptedError')
else:
self.foundExecutable = False
self.raiseAMessage('not found executable '+self.executable,'ExceptedError')
if self.preExec is not None:
if '~' in self.preExec:
self.preExec = os.path.expanduser(self.preExec)
abspath = os.path.abspath(self.preExec)
if os.path.exists(abspath):
self.preExec = abspath
else:
self.foundPreExec = False
self.raiseAMessage('not found preexec '+self.preExec,'ExceptedError')
self.code = Code.CodeInterfaces.returnCodeInterface(self.subType,self)
self.code.readMoreXML(xmlNode) #TODO figure out how to handle this with InputData
self.code.setInputExtension(list(a[0].strip('.') for b in (c for c in self.clargs['input'].values()) for a in b))
self.code.addInputExtension(list(a.strip('.') for b in (c for c in self.fargs ['input'].values()) for a in b))
self.code.addDefaultExtension()
def _localReadMoreXML(self,xmlNode):
"""
Function to read the portion of the xml input that belongs to this specialized class
and initialize some stuff based on the inputs got
@ In, xmlNode, xml.etree.Element, Xml element node
@ Out, None
"""
# paramInput = RavenOutput.getInputSpecification()()
# paramInput.parseNode(xmlNode)
#check if in dynamic mode; default is False
dynamicNode = xmlNode.find('dynamic')
if dynamicNode is not None:
#could specify as true/false or just have the node present
text = dynamicNode.text
if text is not None:
if text not in utils.stringsThatMeanFalse():
self.dynamic = True
else:
self.dynamic = True
numberOfSources = 0
for child in xmlNode:
#if dynamic, accept a single file as <File ID="1" name="myOut.xml">
#if not dynamic, accept a list of files
if child.tag == 'File':
numberOfSources += 1
if 'name' not in child.attrib.keys():
self.raiseAnError(IOError,'Each "File" must have an associated "name"; missing for',child.tag,child.text)
#make sure you provide an ID and a file name
if 'ID' not in child.attrib.keys():
id = 0
while id in self.files.keys():
id += 1
self.raiseAWarning(IOError,'Each "File" entry must have an associated "ID"; missing for',child.tag,child.attrib['name'],'so ID is set to',id)
else:
#assure ID is a number, since it's going into a data object
id = child.attrib['ID']
try:
id = float(id)
except ValueError:
self.raiseAnError(IOError,'ID for "'+child.text+'" is not a valid number:',id)
#if already used, raise an error
if id in self.files.keys():
self.raiseAnError(IOError,'Multiple File nodes have the same ID:',child.attrib('ID'))
#store id,filename pair
self.files[id] = {'name':child.attrib['name'].strip(), 'fileObject':None, 'paths':{}}
#user provides loading information as <output name="variablename">ans|pearson|x</output>
for cchild in child:
if cchild.tag == 'output':
#make sure you provide a label for this data array
if 'name' not in cchild.attrib.keys():
self.raiseAnError(IOError,'Must specify a "name" for each "output" block! Missing for:',cchild.text)
varName = cchild.attrib['name'].strip()
if varName in self.files[id]['paths'].keys():
self.raiseAnError(IOError,'Multiple "output" blocks for "%s" have the same "name":' %self.files[id]['name'],varName)
self.files[id]['paths'][varName] = cchild.text.strip()
#if dynamic, only one File can be specified currently; to fix this, how do you handle different-lengthed times in same data object?
if self.dynamic and numberOfSources > 1:
self.raiseAnError(IOError,'For Dynamic reading, only one "File" node can be specified! Got',numberOfSources,'nodes.')
# check there are entries for each
if len(self.files)<1:
self.raiseAWarning('No files were specified to read from! Nothing will be done...')
# if no outputs listed, remove file from list and warn
toRemove=[]
for id,fileDict in self.files.items():
if len(fileDict['paths'])<1:
self.raiseAWarning('No outputs were specified for File with ID "%s"! No extraction will be performed for this file...' %str(id))
toRemove.append(id)
for rem in toRemove:
del self.files[id]