def addGroup(self,rlz):
"""
Function to add a group into the database
@ In, groupName, string, group name
@ In, attributes, dict, dictionary of attributes that must be added as metadata
@ In, source, File object, data source (for example, csv file)
@ Out, None
"""
parentID = rlz.get("RAVEN_parentID",[None])[0]
prefix = rlz.get("prefix")
groupName = str(prefix if utils.isSingleValued(prefix) else prefix[0])
if parentID:
#If Hierarchical structure, firstly add the root group
if not self.firstRootGroup or parentID == "None":
self.__addGroupRootLevel(groupName,rlz)
self.firstRootGroup = True
self.type = 'DET'
else:
# Add sub group in the Hierarchical structure
self.__addSubGroup(groupName,rlz)
else:
# Parallel structure (always root level)
self.__addGroupRootLevel(groupName,rlz)
self.firstRootGroup = True
self.type = 'MC'
self.__updateFileLevelInfoDatasets()
self.h5FileW.flush()
def _selectiveRealization(self, rlz):
"""
Uses "options" parameters from input to select part of the collected data
@ In, rlz, dict, {var:val} format (see addRealization)
@ Out, rlz, dict, {var:val} modified
"""
# TODO this could be much more efficient on the parallel (finalizeCodeOutput) than on serial
# TODO someday needs to be implemented for when ND data is collected! For now, use base class.
# TODO externalize it in the DataObject base class
toRemove = []
for var, val in rlz.items():
if var in self.protectedTags:
continue
# only modify it if it is not already scalar
if not utils.isSingleValued(val):
# treat inputs, outputs differently TODO this should extend to per-variable someday
## inputs
if var in self._inputs:
method, indic = self._selectInput
# pivot variables are included here in "else"; remove them after they're used in operators
else:
continue
if method in ['inputRow']:
# zero-d xarrays give false behavior sometimes
# TODO formatting should not be necessary once standardized history,float realizations are established
if type(val) == list:
val = np.array(val)
elif type(val).__name__ == 'DataArray':
val = val.values
# FIXME this is largely a biproduct of old length-one-vector approaches in the deprecataed data objects
if val.size == 1:
rlz[var] = float(val)
else:
rlz[var] = float(val[indic])
elif method in ['inputPivotValue']:
pivotParam = self.getDimensions(var)
assert (len(pivotParam) == 1
) # TODO only handle History for now
pivotParam = pivotParam[var][0]
idx = (np.abs(rlz[pivotParam] - indic)).argmin()
rlz[var] = rlz[var][idx]
elif method == 'operator':
if indic == 'max':
rlz[var] = float(val.max())
elif indic == 'min':
rlz[var] = float(val.min())
elif indic in ['mean', 'expectedValue', 'average']:
rlz[var] = float(val.mean())
# otherwise, leave it alone
return rlz
]
#populating these in this order tests adding new entries to the front (0.0003), back (0.9), and middle (0.85),
# as well as adding matches in the front (0.00029...), back (0.90...1), and middle (0.79...)
desired = [0.0003, 0.002, 0.8, 0.85, 0.9]
sortedList = []
for x in toPopulate:
sortedList, index, match = utils.getRelativeSortedListEntry(sortedList,
x,
tol=1e-6)
checkArray('Maintaining sorted list', sortedList, desired)
##########################
# TYPE CHECKING #
##########################
# isSingleValued
checkAnswer('isSingleValued -1', utils.isSingleValued(-1), True)
checkAnswer('isSingleValued 0', utils.isSingleValued(0), True)
checkAnswer('isSingleValued 1', utils.isSingleValued(1), True)
checkAnswer('isSingleValued 1e200', utils.isSingleValued(1e200), True)
checkAnswer('isSingleValued 1e-200', utils.isSingleValued(1e-200), True)
checkAnswer('isSingleValued -1e200', utils.isSingleValued(-1e200), True)
checkAnswer('isSingleValued 3.14', utils.isSingleValued(3.14), True)
checkAnswer('isSingleValued "hombre"', utils.isSingleValued('hombre'), True)
checkAnswer('isSingleValued None', utils.isSingleValued(None), True)
checkAnswer('isSingleValued True', utils.isSingleValued(True), True)
checkAnswer('isSingleValued False', utils.isSingleValued(False), True)
checkAnswer('isSingleValued long',
utils.isSingleValued(123456789012345678901234567890), True)
checkAnswer('isSingleValued inf notok',
utils.isSingleValued(np.inf, nanOk=False), False)
def modifyOrAdd(self,modiDictionary={},save=True, allowAdd = False):
"""
modiDictionary a dict of dictionaries of the required addition or modification
{"variableToChange":value }
@ In, modiDictionary, dict, dictionary of variables to modify
syntax:
{'Node|SubNode|SubSubNode:value1','Node|SubNode|SubSubNode@attribute:attributeValue|SubSubSubNode':value2
'Node|SubNode|SubSubNode@attribute':value3}
TODO: handle added XML nodes
@ In, save, bool, optional, True if the original tree needs to be saved
@ In, allowAdd, bool, optional, True if the nodes that are not found should be added (additional piece of input)
@ Out, returnElement, xml.etree.ElementTree.Element, the tree that got modified
"""
if save:
returnElement = copy.deepcopy(self.tree) #make a copy if save is requested
else:
returnElement = self.tree #otherwise return the original modified
for fullNode, val in modiDictionary.items():
# might be comma-separated ("fully correlated") variables
nodes = [x.strip() for x in fullNode.split(',')]
for node in nodes:
# make sure node is XML-tree-parsable
if "|" not in node:
raise IOError(self.printTag+' ERROR: the variable '+node.strip()+' does not contain "|" separator and can not be handled!!')
changeTheNode = True
allowAddNodes, allowAddNodesPath = [], OrderedDict()
if "@" in node:
# there are attributes that are needed to locate the node
splittedComponents = node.split("|")
# check the first
pathNode = './'
attribName = ''
for cnt, subNode in enumerate(splittedComponents):
splittedComp = subNode.split("@")
component = splittedComp[0]
attribPath = ""
attribConstruct = OrderedDict()
if "@" in subNode:
# more than an attribute locator
for attribComp in splittedComp[1:]:
attribValue = None
if ":" in attribComp.strip():
# it is a locator
attribName = attribComp.split(":")[0].strip()
attribValue = attribComp.split(":")[1].strip()
attribPath +='[@'+attribName+('="'+attribValue+'"]')
else:
# it is actually the attribute that needs to be changed
# check if it is the last component
if cnt+1 != len(splittedComponents):
raise IOError(self.printTag+' ERROR: the variable '+node.strip()+' follows the syntax "Node|SubNode|SubSubNode@attribute"'+
' but the attribute is not the last component. Please check your input!')
attribName = attribComp.strip()
attribPath +='[@'+attribName+']'
if allowAdd:
attribConstruct[attribName] = attribValue
pathNode += "/" + component.strip()+attribPath
if allowAdd:
if len(returnElement.findall(pathNode)) > 0:
allowAddNodes.append(pathNode)
else:
allowAddNodes.append(None)
allowAddNodesPath[component.strip()] = attribConstruct
if pathNode.endswith("]") and list(attribConstruct.values())[-1] is None:
changeTheNode = False
else:
changeTheNode = True
else:
# there are no attributes that are needed to track down the node to change
pathNode = './/' + node.replace("|","/").strip()
if allowAdd:
pathNodeTemp = './'
for component in node.replace("|","/").split("/"):
pathNodeTemp += '/'+component
if len(returnElement.findall(pathNodeTemp)) > 0:
allowAddNodes.append(pathNodeTemp)
else:
allowAddNodes.append(None)
allowAddNodesPath[component.strip()] = None
# look for the node with XPath directives
foundNodes = returnElement.findall(pathNode)
if len(foundNodes) > 1:
raise IOError(self.printTag+' ERROR: multiple nodes have been found corresponding to path -> '+node.strip()+'. Please use the attribute identifier "@" to nail down to a specific node !!')
if len(foundNodes) == 0 and not allowAdd:
raise IOError(self.printTag+' ERROR: no node has been found corresponding to path -> '+node.strip()+'. Please check the input!!')
if len(foundNodes) == 0:
# this means that the allowAdd is true (=> no error message has been raised)
indexFirstUnknownNode = allowAddNodes.index(None)
if indexFirstUnknownNode == 0:
raise IOError(self.printTag+' ERROR: at least the main XML node should be present in the RAVEN template input -> '+node.strip()+'. Please check the input!!')
getFirstElement = returnElement.findall(allowAddNodes[indexFirstUnknownNode-1])[0]
for i in range(indexFirstUnknownNode,len(allowAddNodes)):
nodeWithAttributeName = list(allowAddNodesPath.keys())[i]
if not allowAddNodesPath[nodeWithAttributeName]:
subElement = ET.Element(nodeWithAttributeName)
else:
subElement = ET.Element(nodeWithAttributeName, attrib=allowAddNodesPath[nodeWithAttributeName])
getFirstElement.append(subElement)
getFirstElement = subElement
# in the event of vector entries, handle those here
if utils.isSingleValued(val):
val = str(val).strip()
else:
if len(val.shape) > 1:
raise IOError(self.printTag+'ERROR: RAVEN interface is not prepared to handle matrix value passing yet!')
val = ','.join(str(i) for i in val)
if changeTheNode:
subElement.text = val
else:
subElement.attrib[attribConstruct.keys()[-1]] = val
else:
nodeToChange = foundNodes[0]
pathNode = './/'
# in the event of vector entries, handle those here
if utils.isSingleValued(val):
val = str(val).strip()
else:
if len(val.shape) > 1:
raise IOError(self.printTag+'ERROR: RAVEN interface is not prepared to handle matrix value passing yet!')
val = ','.join(str(i) for i in val)
if changeTheNode:
nodeToChange.text = val
else:
nodeToChange.attrib[attribName] = val
return returnElement
def _selectiveRealization(self, rlz):
"""
Uses "options" parameters from input to select part of the collected data
@ In, rlz, dict, {var:val} format (see addRealization)
@ Out, rlz, dict, {var:val} modified
"""
# TODO this could be much more efficient on the parallel (finalizeCodeOutput) than on serial
# TODO costly for loop
# TODO overwrites rlz by reference; is this okay?
# data was previously formatted by _formatRealization
# then select the point we want
toRemove = []
for var, val in rlz.items():
if var in self.protectedTags:
continue
# only modify it if it is not already scalar
if not utils.isSingleValued(val):
# treat inputs, outputs differently TODO this should extend to per-variable someday
## inputs
if var in self._inputs:
method, indic = self._selectInput
elif var in self._outputs or var in self._metavars:
# TODO where does metadata get picked from? Seems like output fits best?
method, indic = self._selectOutput
# pivot variables are included here in "else"; remove them after they're used in operators
else:
toRemove.append(var)
continue
if method in ['inputRow', 'outputRow']:
# zero-d xarrays give false behavior sometimes
# TODO formatting should not be necessary once standardized history,float realizations are established
if type(val) == list:
val = np.array(val)
elif type(val).__name__ == 'DataArray':
val = val.values
# FIXME this is largely a biproduct of old length-one-vector approaches in the deprecataed data objects
if val.size == 1:
rlz[var] = float(val)
else:
rlz[var] = float(val[indic])
elif method in ['inputPivotValue', 'outputPivotValue']:
pivotParam = self.getDimensions(var)
assert (len(pivotParam) == 1
) # TODO only handle History for now
pivotParam = pivotParam[var][0]
idx = (np.abs(rlz[pivotParam] - indic)).argmin()
rlz[var] = rlz[var][idx]
# if history is dataarray -> not currently possible, but keep for when it's needed
#if type(rlz[var]).__name__ == 'DataArray':
# rlz[var] = float(val.sel(**{pivotParam:indic, 'method':b'nearest'})) #casting as str not unicode
# TODO allowing inexact matches; it's finding the nearest
elif method == 'operator':
if indic == 'max':
rlz[var] = float(val.max())
elif indic == 'min':
rlz[var] = float(val.min())
elif indic in ['mean', 'expectedValue', 'average']:
rlz[var] = float(val.mean())
# otherwise, leave it alone
for var in toRemove:
del rlz[var]
return rlz