def printInput(self,outfile=None):
"""
Method to print out the new input
@ In, outfile, string, optional, output file root
@ Out, None
"""
# 4 sub levels maximum
def printSubLevels(xmlnode,IOfile,indentMultiplier):
IOfile.write(b' '*indentMultiplier+b'[./'+toBytes(xmlnode.tag)+b']\n')
for string in xmlnode.tail if xmlnode.tail else []:
IOfile.write(b' '*indentMultiplier+string+b'\n')
for key in xmlnode.attrib.keys():
IOfile.write(b' '*indentMultiplier+toBytes(key)+b' = '+toBytes(toStrish(xmlnode.attrib[key]))+b'\n')
if outfile==None:
outfile =self.inputfile
IOfile = open(outfile,'wb')
for child in self.root:
IOfile.write(b'['+toBytes(child.tag)+b']\n')
if child.tail:
for string in child.tail:
IOfile.write(b' '+string+b'\n')
for key in child.attrib.keys():
IOfile.write(b' '+toBytes(key)+b' = '+toBytes(toStrish(child.attrib[key]))+b'\n')
for childChild in child:
printSubLevels(childChild,IOfile,1)
for childChildChild in childChild:
printSubLevels(childChildChild,IOfile,2)
for childChildChildChild in childChildChild:
printSubLevels(childChildChildChild,IOfile,3)
IOfile.write(b' [../]\n')
IOfile.write(b' [../]\n')
IOfile.write(b' [../]\n')
IOfile.write(b'[]\n')
def printSubLevels(xmlnode, IOfile, indentMultiplier):
IOfile.write(b' ' * indentMultiplier + b'[./' +
toBytes(xmlnode.tag) + b']\n')
for string in xmlnode.tail if xmlnode.tail else []:
IOfile.write(b' ' * indentMultiplier + string + b'\n')
for key in xmlnode.attrib.keys():
IOfile.write(b' ' * indentMultiplier + toBytes(key) +
b' = ' + toBytes(toStrish(xmlnode.attrib[key])) +
b'\n')
def __updateGroupLists(self,groupName, parentName):
"""
Utility method to update the group lists
@ In, groupName, str, the new group added
@ In, parentName, str, the parent name
@ Out, None
"""
if parentName != "/":
self.allGroupPaths.append(utils.toBytes(parentName) + b"/" + utils.toBytes(groupName))
self.allGroupEnds.append(True)
else:
self.allGroupPaths.append(b"/" + utils.toBytes(groupName))
self.allGroupEnds.append(True)
def __matchDict(self, dictionary, other):
"""
Method to check the consistency of two dictionaries
Returns true if all the keys and values in other
match all the keys and values in dictionary.
Note that it does not check that all the keys in dictionary
match all the keys and values in other.
@ In, dictionary, dict, first dictionary to check
@ In, other, dict, second dictionary to check
@ Out, returnBool, bool, True if all the keys and values in other match all the keys and values in dictionary.
"""
returnBool = True
for key in other:
if key in dictionary:
#if dictionary[key] != other[key]:
if not compare(dictionary[key], other[key]):
print("Missmatch ", key, repr(dictionary[key]),
repr(other[key]))
returnBool = False
else:
binKey = toBytes(key)
if binKey in dictionary:
if not compare(dictionary[binKey], other[key]):
print("Missmatch_b ", key, dictionary[binKey],
other[key])
returnBool = False
else:
print("No_key ", key, other[key])
returnBool = False
return returnBool
def finalizeCodeOutput(self, command, output, workingDir):
"""
Called by RAVEN to modify output files (if needed) so that they are in a proper form.
In this case, OpenModelica CSV output comes with trailing commas that RAVEN doesn't
like. So we have to strip them. Also, the first line (with the variable names)
has those names enclosed in double quotes (which we have to remove)
@ In, command, string, the command used to run the just ended job
@ In, output, string, the Output name root
@ In, workingDir, string, current working dir
@ Out, destFileName, string, present in case the root of the output file gets changed in this method.
"""
# Make a new temporary file in the working directory and read the lines from the original CSV
# to it, stripping trailing commas in the process.
tempOutputFD, tempOutputFileName = tempfile.mkstemp(dir = workingDir, text = True)
sourceFileName = os.path.join(workingDir, output) # The source file comes in without .csv on it
print('sourcefilename:',sourceFileName)
destFileName = sourceFileName.replace('rawout~', 'out~') # When fix the CSV, change rawout~ to out~
sourceFileName += '.csv'
inputFile = open(sourceFileName)
for line in inputFile:
# Line ends with a comma followed by a newline
#XXX toBytes seems to be needed here in python3, despite the text = True
os.write(tempOutputFD, utils.toBytes(line.replace('"','').strip().strip(',') + '\n'))
inputFile.close()
os.close(tempOutputFD)
shutil.move(tempOutputFileName, destFileName + '.csv')
return destFileName # Return the name without the .csv on it...RAVEN will add it
def __updateDict(self, dictionary, other):
"""
Add all the keys and values in other into dictionary
@ In, dictionary, dict, dictionary that needs to be updated
@ In, other, dict, dictionary from which the valued need to be taken
@ Out, None
"""
for key in other:
if key in dictionary: dictionary[key] = other[key]
else:
binKey = toBytes(key)
if binKey in dictionary: dictionary[binKey] = other[key]
else: dictionary[key] = other[key]
def __findInXML(self, element, name):
"""
Checks if there is a tag with name or binary name in
element, and returns the (found,actual_name)
@ In, element, xml.etree.ElementTree.Element, element where the 'name' needs to be found
@ In, name, string, name of the node to be found
@ Out, returnTuple, tuple, response. returnTuple[0] bool (True if found) and returnTuple[1] string (binary name in the element)
"""
returnTuple = None
if element.find(name) is not None:
returnTuple = (True, name)
else:
binaryName = toBytes(name)
if element.find(binaryName) is not None:
returnTuple = (True, binaryName)
else:
returnTuple = (False, None)
return returnTuple
def __isGroup(self,name,obj):
"""
Function to check if an object name is of type "group". If it is, the function stores
its name into the "self.allGroupPaths" list and update the dictionary "self.allGroupEnds"
@ In, name, string, object name
@ In, obj, object, the object itself
@ Out, None
"""
if isinstance(obj,h5.Group):
self.allGroupPaths.append(utils.toBytes(name))
try:
self.allGroupEnds.append(obj.attrs["endGroup"])
except KeyError:
self.allGroupEnds.append(True)
if "rootname" in obj.attrs:
self.parentGroupName = name
self.raiseAWarning('not found attribute endGroup in group ' + name + '.Set True.')
return
def addGroupInit(self, groupName, attributes=None):
"""
Function to add an empty group to the database
This function is generally used when the user provides a rootname in the input.
It uses the groupName + it appends the date and time.
@ In, groupName, string, group name
@ In, attributes, dict, optional, dictionary of attributes that must be added as metadata (None by default)
@ Out, None
"""
attribs = {} if attributes is None else attributes
groupNameInit = groupName + "_" + datetime.now().strftime(
"%m-%d-%Y-%H-%S")
for index in range(len(self.allGroupPaths)):
comparisonName = utils.toString(self.allGroupPaths[index])
splittedPath = comparisonName.split('/')
if len(splittedPath) > 0:
if groupNameInit in splittedPath[0]:
alphabetCounter, movingCounter = 0, 0
asciiAlphabet = list(string.ascii_uppercase)
prefixLetter = ''
while True:
testGroup = groupNameInit + "_" + prefixLetter + asciiAlphabet[
alphabetCounter]
if testGroup not in self.allGroupPaths:
groupNameInit = utils.toString(testGroup)
break
alphabetCounter += 1
if alphabetCounter >= len(asciiAlphabet):
prefix = asciiAlphabet[movingCounter]
alphabetCounter = 0
movingCounter += 1
break
self.parentGroupName = "/" + groupNameInit
# Create the group
grp = self.h5FileW.create_group(groupNameInit)
# Add metadata
grp.attrs.update(attribs)
grp.attrs['rootname'] = True
grp.attrs['endGroup'] = False
grp.attrs[b'groupName'] = groupNameInit
self.allGroupPaths.append(utils.toBytes("/" + groupNameInit))
self.allGroupEnds.append(False)
self.__updateFileLevelInfoDatasets()
self.h5FileW.flush()
def __addSubGroup(self,groupName,attributes,source):
"""
Function to add a group into the database (Hierarchical)
@ In, groupName, string, group name
@ In, attributes, dict, dictionary of attributes that must be added as metadata
@ In, source, File object, source data
@ Out, None
"""
for index in xrange(len(self.allGroupPaths)):
comparisonName = self.allGroupPaths[index]
splittedPath=comparisonName.split('/')
for splgroup in splittedPath:
if groupName == splgroup and splittedPath[0] == self.parentGroupName:
self.raiseAnError(IOError,"Group named " + groupName + " already present in database " + self.name + ". new group " + groupName + " is equal to old group " + comparisonName)
if source['type'] == 'csv':
# Source in CSV format
f = open(source['name'],'rb')
# Retrieve the headers of the CSV file
headers = f.readline().split(b",")
# Load the csv into a numpy array(n time steps, n parameters)
data = np.loadtxt(f,dtype='float',delimiter=',',ndmin=2)
# Check if the parent attribute is not null # In this case append a subgroup to the parent group
# Otherwise => it's the main group
parentID = None
if 'metadata' in attributes.keys():
if 'parentID' in attributes['metadata'].keys():
parentID = attributes['metadata']['parentID']
else:
if 'parentID' in attributes.keys():
parentID = attributes['parentID']
if parentID:
parentName = parentID
else:
self.raiseAnError(IOError,'NOT FOUND attribute <parentID> into <attributes> dictionary')
# Find parent group path
if parentName != '/':
parentGroupName = self.__returnParentGroupPath(parentName)
else:
parentGroupName = parentName
# Retrieve the parent group from the HDF5 database
if parentGroupName in self.h5FileW:
grp = self.h5FileW.require_group(parentGroupName)
else:
# try to guess the parentID from the file name
head,tail = os.path.split(os.path.dirname(source['name']))
testParentName = self.__returnParentGroupPath(tail[:-2])
if testParentName in self.h5FileW:
grp = self.h5FileW.require_group(testParentName)
else:
closestGroup = difflib.get_close_matches(parentName, self.allGroupPaths, n=1, cutoff=0.01)
errorString = ' NOT FOUND parent group named "' + str(parentName)+'" for loading file '+str(source['name'])
errorString+= '\n Tried '+str(tail[:-2])+ ' but not found as well. All group paths are:\n -'+'\n -'.join(self.allGroupPaths)
errorString+= '\n Closest parent group found is "'+str(closestGroup[0] if len(closestGroup) > 0 else 'None')+'"!'
self.raiseAnError(ValueError,errorString)
# The parent group is not the endgroup for this branch
self.allGroupEnds[parentGroupName] = False
grp.attrs["EndGroup"] = False
self.raiseAMessage('Adding group named "' + groupName + '" in Database "'+ self.name +'"')
# Create the sub-group
sgrp = grp.create_group(groupName)
# Create data set in this new group
sgrp.create_dataset(groupName+"_data", dtype="float", data=data)
# Add the metadata
sgrp.attrs["outputSpaceHeaders" ] = headers
sgrp.attrs["nParams" ] = data[0,:].size
sgrp.attrs["parent" ] = "root"
sgrp.attrs["startTime"] = data[0,0]
sgrp.attrs["end_time" ] = data[data[:,0].size-1,0]
sgrp.attrs["nTimeSteps" ] = data[:,0].size
sgrp.attrs["EndGroup" ] = True
sgrp.attrs["sourceType"] = source['type']
if source['type'] == 'csv':
sgrp.attrs["sourceFile"] = source['name']
# add metadata if present
for attr in attributes.keys():
if attr == 'metadata':
if 'SampledVars' in attributes['metadata'].keys():
inpHeaders = []
inpValues = []
for inkey, invalue in attributes['metadata']['SampledVars'].items():
if inkey not in headers:
inpHeaders.append(utils.toBytes(inkey))
inpValues.append(invalue)
if len(inpHeaders) > 0:
sgrp.attrs[b'inputSpaceHeaders'] = inpHeaders
sgrp.attrs[b'inputSpaceValues' ] = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(inpValues))))
#Files objects are not JSON serializable, so we have to cover that.
#this doesn't cover all possible circumstance, but it covers the DET case.
if attr == 'inputFile' and isinstance(attributes[attr][0],Files.File):
objectToConvert = list(a.__getstate__() for a in attributes[attr])
else:
objectToConvert = mathUtils.convertNumpyToLists(attributes[attr])
converted = json.dumps(objectToConvert)
if converted and attr != 'name':
sgrp.attrs[utils.toBytes(attr)]=converted
else:
pass
# The sub-group is the new ending group
if parentGroupName != "/":
self.allGroupPaths.append(parentGroupName + "/" + groupName)
self.allGroupEnds[parentGroupName + "/" + groupName] = True
else:
self.allGroupPaths.append("/" + groupName)
self.allGroupEnds["/" + groupName] = True
return
def addGroupDataObjects(self,groupName,attributes,source,upGroup=False,specificVars=None):
"""
Function to add a data (class DataObjects) or Dictionary into the Database
@ In, groupName, string, group name
@ In, attributes, dict, dictionary of attributes that must be added as metadata
@ In, source, dataObject, source data
@ In, upGroup, bool, optional, updated group?
@ In, specificVars, list(str), if not None then indicates a selective list of variables to include in DB
@ Out, None
"""
if not upGroup:
for index in xrange(len(self.allGroupPaths)):
comparisonName = self.allGroupPaths[index]
splittedPath=comparisonName.split('/')
for splgroup in splittedPath:
if groupName == splgroup and splittedPath[0] == self.parentGroupName:
found = True
while found:
if groupName in splittedPath:
found = True
else:
found = False
groupName = groupName + "_"+ str(index)
#self.raiseAnError(IOError,"Group named " + groupName + " already present in database " + self.name + ". new group " + groupName + " is equal to old group " + comparisonName)
parentName = self.parentGroupName.replace('/', '')
# Create the group
if parentName != '/':
parentGroupName = self.__returnParentGroupPath(parentName)
# Retrieve the parent group from the HDF5 database
if parentGroupName in self.h5FileW:
parentGroupObj = self.h5FileW.require_group(parentGroupName)
else:
self.raiseAnError(ValueError,'NOT FOUND group named ' + parentGroupObj)
else:
parentGroupObj = self.h5FileW
if type(source['name']) == dict:
# create the group
if upGroup:
groups = parentGroupObj.require_group(groupName)
del groups[groupName+"_data"]
else:
groups = parentGroupObj.create_group(groupName)
groups.attrs[b'mainClass' ] = b'PythonType'
groups.attrs[b'sourceType'] = b'Dictionary'
# I keep this structure here because I want to maintain the possibility to add a whatever dictionary even if not prepared and divided into output and input sub-sets. A.A.
# use ONLY the subset of variables if requested
if set(['inputSpaceParams']).issubset(set(source['name'].keys())):
sourceInputs = source['name']['inputSpaceParams'].keys()
if specificVars is not None:
inputHeaders = list(var for var in sourceInputs if var in specificVars)
else:
inputHeaders = sourceInputs
inputHeaders = list(utils.toBytesIterative(inputHeaders))
groups.attrs[b'inputSpaceHeaders' ] = inputHeaders
if specificVars is not None:
inputValues = list(source['name']['inputSpaceParams'][var] for var in sourceInputs if var in specificVars)
else:
inputValues = source['name']['inputSpaceParams'].values()
inputValues = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(utils.toBytesIterative(inputValues)))))
groups.attrs[b'inputSpaceValues' ] = inputValues
if set(['outputSpaceParams']).issubset(set(source['name'].keys())):
if specificVars is not None:
outDict = dict((k,v) for k,v in source['name']['outputSpaceParams'].items() if k in specificVars)
else:
outDict = source['name']['outputSpaceParams']
else:
if specificVars is not None:
outDict = dict((key,value) for (key,value) in source['name'].iteritems() if key not in ['inputSpaceParams'] and key in specificVars)
else:
outDict = dict((key,value) for (key,value) in source['name'].iteritems() if key not in ['inputSpaceParams'])
outHeaders = utils.toBytesIterative(list(outDict.keys()))
outValues = utils.toBytesIterative(list(outDict.values()))
groups.attrs[b'nParams' ] = len(outHeaders)
groups.attrs[b'outputSpaceHeaders'] = outHeaders
groups.attrs[b'EndGroup' ] = True
groups.attrs[b'parentID' ] = parentName
maxSize = 0
for value in outValues:
if type(value) == np.ndarray or type(value).__name__ == 'c1darray':
if maxSize < value.size:
actualOne = np.asarray(value).size
elif type(value) in [int,float,bool,np.float64,np.float32,np.float16,np.int64,np.int32,np.int16,np.int8,np.bool8]:
actualOne = 1
else:
self.raiseAnError(IOError,'The type of the dictionary parameters must be within float,bool,int,numpy.ndarray.Got '+type(value).__name__)
if maxSize < actualOne:
maxSize = actualOne
groups.attrs[b'nTimeSteps' ] = maxSize
dataout = np.zeros((maxSize,len(outHeaders)))
for index in range(len(outHeaders)):
if type(outValues[index]) == np.ndarray or type(value).__name__ == 'c1darray':
dataout[0:outValues[index].size,index] = np.ravel(outValues[index])[:]
else:
dataout[0,index] = outValues[index]
# create the data set
groups.create_dataset(groupName + "_data", dtype="float", data=dataout)
# add metadata if present
for attr in attributes.keys():
objectToConvert = mathUtils.convertNumpyToLists(attributes[attr])
converted = json.dumps(objectToConvert)
if converted and attr != 'name':
groups.attrs[utils.toBytes(attr)]=converted
if parentGroupName != "/":
self.allGroupPaths.append(parentGroupName + "/" + groupName)
self.allGroupEnds[parentGroupName + "/" + groupName] = True
else:
self.allGroupPaths.append("/" + groupName)
self.allGroupEnds["/" + groupName] = True
else:
# Data(structure)
# Retrieve the headers from the data (inputs and outputs)
inpParams = source['name'].getInpParametersValues().keys()
outParams = source['name'].getOutParametersValues().keys()
if specificVars is not None:
headersIn = list(v for v in inpParams if v in specificVars)
headersOut = list(v for v in outParams if v in specificVars)
else:
headersIn = list(inpParams)
headersOut = list(outParams)
# for a "HistorySet" type we create a number of groups = number of HistorySet (compatibility with loading structure)
if specificVars is not None:
dataIn = list(source['name'].getInpParametersValues()[v] for v in inpParams if v in specificVars)
dataOut = list(source['name'].getOutParametersValues()[v] for v in outParams if v in specificVars)
else:
dataIn = list(source['name'].getInpParametersValues().values())
dataOut = list(source['name'].getOutParametersValues().values())
# FIXME unused, but left commented because I'm not sure why they're unused. PT
#headersInUnstructured = list(source['name'].getInpParametersValues(self,unstructuredInputs=True).keys())
#dataInUnstructured = list(source['name'].getInpParametersValues(self,unstructuredInputs=True).values())
metadata = source['name'].getAllMetadata()
if source['name'].type in ['HistorySet','PointSet']:
groups = []
if 'HistorySet' in source['name'].type:
nruns = len(dataIn)
else:
nruns = dataIn[0].size
for run in range(nruns):
if upGroup:
groups.append(parentGroupObj.require_group(groupName + b'|' +str(run)))
if (groupName + "_data") in groups[run]:
del groups[run][groupName+"_data"]
else:
groups.append(parentGroupObj.create_group(groupName + '|' +str(run)))
groups[run].attrs[b'sourceType'] = utils.toBytes(source['name'].type)
groups[run].attrs[b'mainClass' ] = b'DataObjects'
groups[run].attrs[b'EndGroup' ] = True
groups[run].attrs[b'parentID' ] = parentName
if source['name'].type == 'HistorySet':
groups[run].attrs[b'inputSpaceHeaders' ] = [utils.toBytes(list(dataIn[run].keys())[i]) for i in range(len(dataIn[run].keys()))]
groups[run].attrs[b'outputSpaceHeaders'] = [utils.toBytes(list(dataOut[run].keys())[i]) for i in range(len(dataOut[run].keys()))]
json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(dataIn[run].values()))))
groups[run].attrs[b'inputSpaceValues' ] = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(dataIn[run].values()))))
groups[run].attrs[b'nParams' ] = len(dataOut[run].keys())
#collect the outputs
dataout = np.zeros((next(iter(dataOut[run].values())).size,len(dataOut[run].values())))
for param in range(len(dataOut[run].values())):
dataout[:,param] = list(dataOut[run].values())[param][:]
groups[run].create_dataset(groupName +'|' +str(run)+"_data" , dtype="float", data=dataout)
groups[run].attrs[b'nTimeSteps' ] = next(iter(dataOut[run].values())).size
else:
groups[run].attrs[b'inputSpaceHeaders' ] = [utils.toBytes(headersIn[i]) for i in range(len(headersIn))]
groups[run].attrs[b'outputSpaceHeaders'] = [utils.toBytes(headersOut[i]) for i in range(len(headersOut))]
groups[run].attrs[b'inputSpaceValues' ] = json.dumps([list(utils.toListFromNumpyOrC1arrayIterative(np.atleast_1d(np.array(dataIn[x][run])).tolist())) for x in range(len(dataIn))])
groups[run].attrs[b'nParams' ] = len(headersOut)
groups[run].attrs[b'nTimeSteps' ] = 1
#collect the outputs
dataout = np.zeros((1,len(dataOut)))
for param in range(len(dataOut)):
dataout[0,param] = dataOut[param][run]
groups[run].create_dataset(groupName +'|' +str(run)+"_data", dtype="float", data=dataout)
# add metadata if present
for attr in attributes.keys():
objectToConvert = mathUtils.convertNumpyToLists(attributes[attr])
converted = json.dumps(objectToConvert)
if converted and attr != 'name':
groups[run].attrs[utils.toBytes(attr)]=converted
for attr in metadata.keys():
if len(metadata[attr]) == nruns:
toProcess = metadata[attr][run]
else:
toProcess = metadata[attr]
if type(toProcess).__name__ == 'list' and 'input' in attr.lower() and isinstance(toProcess[0],Files.File):
objectToConvert = list(a.__getstate__() for a in toProcess)
elif isinstance(toProcess,Files.File):
objectToConvert =toProcess.__getstate__()
else:
objectToConvert = mathUtils.convertNumpyToLists(toProcess)
converted = json.dumps(objectToConvert)
if converted and attr != 'name':
groups[run].attrs[utils.toBytes(attr)]=converted
if parentGroupName != "/":
self.allGroupPaths.append(parentGroupName + "/" + groupName + '|' +str(run))
self.allGroupEnds[parentGroupName + "/" + groupName + '|' +str(run)] = True
else:
self.allGroupPaths.append("/" + groupName + '|' +str(run))
self.allGroupEnds["/" + groupName + '|' +str(run)] = True
else:
self.raiseAnError(IOError,'The function addGroupDataObjects accepts Data(s) or dictionaries as inputs only!!!!!')
def __addGroupRootLevel(self,groupName,attributes,source,upGroup=False):
"""
Function to add a group into the database (root level)
@ In, groupName, string, group name
@ In, attributes, dict, dictionary of attributes that must be added as metadata
@ In, source, File object, source file
@ In, upGroup, bool, optional, updated group?
@ Out, None
"""
# Check in the "self.allGroupPaths" list if a group is already present...
# If so, error (Deleting already present information is not desiderable)
if not upGroup:
for index in xrange(len(self.allGroupPaths)):
comparisonName = self.allGroupPaths[index]
splittedPath=comparisonName.split('/')
for splgroup in splittedPath:
if groupName == splgroup and splittedPath[0] == self.parentGroupName:
self.raiseAnError(IOError,"Group named " + groupName + " already present in database " + self.name + ". new group " + groupName + " is equal to old group " + comparisonName)
if source['type'] == 'csv':
# Source in CSV format
f = open(source['name'],'rb')
# Retrieve the headers of the CSV file
firstRow = f.readline().strip(b"\r\n")
#firstRow = f.readline().translate(None,"\r\n")
headers = firstRow.split(b",")
# if there is the alias system, replace the variable name
if 'alias' in attributes.keys():
for aliasType in attributes['alias'].keys():
for var in attributes['alias'][aliasType].keys():
if attributes['alias'][aliasType][var].strip() in headers:
headers[headers.index(attributes['alias'][aliasType][var].strip())] = var.strip()
else:
metadataPresent = True if 'metadata' in attributes.keys() and 'SampledVars' in attributes['metadata'].keys() else False
if not (metadataPresent and var.strip() in attributes['metadata']['SampledVars'].keys()):
self.raiseAWarning('the ' + aliasType +' alias"'+var.strip()+'" has been defined but has not been found among the variables!')
# Load the csv into a numpy array(n time steps, n parameters)
data = np.loadtxt(f,dtype='float',delimiter=',',ndmin=2)
# First parent group is the root name
parentName = self.parentGroupName.replace('/', '')
# Create the group
if parentName != '/':
parentGroupName = self.__returnParentGroupPath(parentName)
# Retrieve the parent group from the HDF5 database
if parentGroupName in self.h5FileW:
rootgrp = self.h5FileW.require_group(parentGroupName)
else:
self.raiseAnError(ValueError,'NOT FOUND group named "' + parentGroupName+'" for loading file '+str(source['name']))
if upGroup:
grp = rootgrp.require_group(groupName)
del grp[groupName+"_data"]
else:
grp = rootgrp.create_group(groupName)
else:
if upGroup:
grp = self.h5FileW.require_group(groupName)
else:
grp = self.h5FileW.create_group(groupName)
self.raiseAMessage('Adding group named "' + groupName + '" in DataBase "'+ self.name +'"')
# Create dataset in this newly added group
grp.create_dataset(groupName+"_data", dtype="float", data=data)
# Add metadata
grp.attrs["outputSpaceHeaders" ] = headers
grp.attrs["nParams" ] = data[0,:].size
grp.attrs["parentID" ] = "root"
grp.attrs["startTime" ] = data[0,0]
grp.attrs["end_time" ] = data[data[:,0].size-1,0]
grp.attrs["nTimeSteps" ] = data[:,0].size
grp.attrs["EndGroup" ] = True
grp.attrs["sourceType" ] = source['type']
if source['type'] == 'csv':
grp.attrs["sourceFile"] = source['name']
for attr in attributes.keys():
if attr == 'metadata':
if 'SampledVars' in attributes['metadata'].keys():
inpHeaders = []
inpValues = []
for inkey, invalue in attributes['metadata']['SampledVars'].items():
if inkey not in headers:
inpHeaders.append(utils.toBytes(inkey))
inpValues.append(invalue)
if len(inpHeaders) > 0:
grp.attrs[b'inputSpaceHeaders'] = inpHeaders
grp.attrs[b'inputSpaceValues' ] = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list( inpValues))))
objectToConvert = mathUtils.convertNumpyToLists(attributes[attr])
for o,obj in enumerate(objectToConvert):
if isinstance(obj,Files.File):
objectToConvert[o]=obj.getFilename()
converted = json.dumps(objectToConvert)
if converted and attr != 'name':
grp.attrs[utils.toBytes(attr)]=converted
#decoded = json.loads(grp.attrs[utils.toBytes(attr)])
if "inputFile" in attributes.keys():
grp.attrs[utils.toString("inputFile")] = utils.toString(" ".join(attributes["inputFile"])) if type(attributes["inputFile"]) == type([]) else utils.toString(attributes["inputFile"])
else:
self.raiseAnError(ValueError,source['type'] + " unknown!")
# Add the group name into the list "self.allGroupPaths" and
# set the relative bool flag into the dictionary "self.allGroupEnds"
if parentGroupName != "/":
self.allGroupPaths.append(parentGroupName + "/" + groupName)
self.allGroupEnds[parentGroupName + "/" + groupName] = True
else:
self.allGroupPaths.append("/" + groupName)
self.allGroupEnds["/" + groupName] = True
def __retrieveDataPointSet(self, attributes):
"""
Function to retrieve a PointSet from the HDF5 database
@ In, attributes, dict, options (metadata must be appended to the root group)
@ Out, tupleVar, tuple, tuple in which the first position is a dictionary of numpy arays (input variable)
and the second is a dictionary of the numpy arrays (output variables).
"""
# Check the outParam variables and the outputPivotVal filters
inParam, outParam, inputRow, outputRow = attributes[
'inParam'], attributes['outParam'], copy.deepcopy(
attributes.get('inputRow', None)), copy.deepcopy(
attributes.get('outputRow', None))
inputPivotVal, outputPivotVal, operator = attributes.get(
'inputPivotValue',
None), attributes.get('outputPivotValue',
None), attributes.get('operator', None)
pivotParameter = attributes.get('pivotParameter', None)
if outParam == 'all': allOutParam = True
else: allOutParam = False
if outputPivotVal != None:
if 'end' in outputPivotVal: outputPivotValEnd = True
else:
outputPivotValEnd, outputPivotVal = False, float(
outputPivotVal)
else:
if operator is None: outputPivotValEnd = True
else: outputPivotValEnd = False
if inputRow == None and inputPivotVal == None: inputRow = 0
if inputRow == None and inputPivotVal == None: inputRow = 0
if inputRow != None:
inputRow = int(inputRow)
if inputRow > 0: inputRow -= 1
if outputRow != None:
outputRow = int(outputRow)
if outputRow > 0: outputRow -= 1
inDict = {}
outDict = {}
metaDict = {}
histList = attributes['HistorySet']
# Retrieve all the associated HistorySet and process them
for i in range(len(histList)):
# Load the data into the numpy array
attributes['history'] = histList[i]
histVar = self.returnHistory(attributes)
#look for pivotParameter
if pivotParameter != None:
pivotIndex = histVar[1]['outputSpaceHeaders'].index(
pivotParameter) if pivotParameter in histVar[1][
'outputSpaceHeaders'] else None
if pivotIndex == None:
self.raiseAnError(
IOError, 'pivotParameter ' + pivotParameter +
' has not been found in history ' +
str(attributes['history']) + '!')
else:
pivotIndex = histVar[1]['outputSpaceHeaders'].index(
"time"
) if "time" in histVar[1]['outputSpaceHeaders'] else None
# if None...default is 0
if pivotIndex == None: pivotIndex = 0
if inputRow > histVar[0][:, 0].size - 1 and inputRow != -1:
self.raiseAnError(
IOError,
'inputRow is greater than number of actual rows in history '
+ str(attributes['history']) + '!')
# check metadata
if 'metadata' in histVar[1].keys():
metaDict[i] = histVar[1]['metadata']
else:
metaDict[i] = None
for key in inParam:
if 'inputSpaceHeaders' in histVar[1]:
inInKey = utils.keyIn(histVar[1]['inputSpaceHeaders'], key)
inOutKey = utils.keyIn(histVar[1]['outputSpaceHeaders'],
key)
if inInKey != None:
ix = histVar[1]['inputSpaceHeaders'].index(inInKey)
if i == 0: inDict[key] = np.zeros(len(histList))
inDict[key][i] = np.atleast_1d(
histVar[1]['inputSpaceValues'][ix])[0]
elif inOutKey != None and inInKey == None:
ix = histVar[1]['outputSpaceHeaders'].index(inOutKey)
if i == 0: inDict[key] = np.zeros(len(histList))
if inputPivotVal != None:
if float(inputPivotVal) > np.max(
histVar[0][:, pivotIndex]) or float(
inputPivotVal) < np.min(
histVar[0][:, pivotIndex]):
self.raiseAnError(
IOError,
'inputPivotVal is out of the min and max for input '
+ key + ' in Database ' + str(self.name) +
'!')
inDict[key][i] = interp1d(
histVar[0][:, pivotIndex],
histVar[0][:, ix],
kind='linear')(float(inputPivotVal))
else:
inDict[key][i] = histVar[0][inputRow, ix]
else:
self.raiseAnError(
IOError,
'the parameter ' + key + ' has not been found')
else:
inKey = utils.keyIn(histVar[1]['outputSpaceHeaders'], key)
if inKey is not None:
ix = histVar[1]['outputSpaceHeaders'].index(inKey)
if i == 0: inDict[key] = np.zeros(len(histList))
if inputPivotVal != None:
if float(inputPivotVal) > np.max(
histVar[0][:, pivotIndex]) or float(
inputPivotVal) < np.min(
histVar[0][:, pivotIndex]):
self.raiseAnError(
IOError,
'inputPivotVal is out of the min and max for input '
+ key + ' in Database ' + str(self.name) +
'!')
inDict[key][i] = interp1d(
histVar[0][:, pivotIndex],
histVar[0][:, ix],
kind='linear')(float(inputPivotVal))
else:
inDict[key][i] = histVar[0][inputRow, ix]
else:
self.raiseAnError(
IOError, 'the parameter ' + key +
' has not been found in ' + str(histVar[1]))
# outputPivotVal end case => PointSet is at the final status
if outputPivotValEnd:
if allOutParam:
for key in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
outDict[key][i] = histVar[0][
-1, histVar[1]['outputSpaceHeaders'].index(key)]
else:
for key in outParam:
if key in histVar[1]['outputSpaceHeaders'] or \
utils.toBytes(key) in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
if key in histVar[1]['outputSpaceHeaders']:
outDict[key][i] = histVar[0][
-1, histVar[1]['outputSpaceHeaders'].
index(key)]
else:
outDict[key][i] = histVar[0][
-1, histVar[1]['outputSpaceHeaders'].
index(utils.toBytes(key))]
else:
self.raiseAnError(
IOError, "the parameter " + str(key) +
" has not been found")
elif outputRow != None:
if outputRow > histVar[0][:, 0].size - 1 and outputRow != -1:
self.raiseAnError(
IOError,
'outputRow is greater than number of actual rows in Database '
+ str(self.name) + '!')
if allOutParam:
for key in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
outDict[key][i] = histVar[0][
outputRow,
histVar[1]['outputSpaceHeaders'].index(key)]
else:
for key in outParam:
if key in histVar[1]['outputSpaceHeaders'] or \
utils.toBytes(key) in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
if key in histVar[1]['outputSpaceHeaders']:
outDict[key][i] = histVar[0][
outputRow,
histVar[1]['outputSpaceHeaders'].index(key
)]
else:
outDict[key][i] = histVar[0][
outputRow,
histVar[1]['outputSpaceHeaders'].
index(utils.toBytes(key))]
else:
self.raiseAnError(
IOError, "the parameter " + str(key) +
" has not been found")
elif operator != None:
if operator not in ['max', 'min', 'average']:
self.raiseAnError(
IOError,
'operator unknown. Available are min,max,average')
if histVar[1]['outputSpaceHeaders']:
for key in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
if operator == 'max':
outDict[key][i] = np.max(
histVar[0]
[:,
histVar[1]['outputSpaceHeaders'].index(key)])
if operator == 'min':
outDict[key][i] = np.min(
histVar[0]
[:,
histVar[1]['outputSpaceHeaders'].index(key)])
if operator == 'average':
outDict[key][i] = np.average(
histVar[0]
[:,
histVar[1]['outputSpaceHeaders'].index(key)])
else:
for key in outParam:
if key in histVar[1]['outputSpaceHeaders'] or \
utils.toBytes(key) in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
if key in histVar[1]['outputSpaceHeaders']:
if operator == 'max':
outDict[key][i] = np.max(
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].
index(key)])
if operator == 'min':
outDict[key][i] = np.min(
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].
index(key)])
if operator == 'average':
outDict[key][i] = np.average(
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].
index(key)])
else:
if operator == 'max':
outDict[key][i] = np.max(
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].
index(utils.toBytes(key))])
if operator == 'min':
outDict[key][i] = np.min(
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].
index(utils.toBytes(key))])
if operator == 'average':
outDict[key][i] = np.average(
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].
index(utils.toBytes(key))])
else:
self.raiseAnError(
IOError, "the parameter " + str(key) +
" has not been found")
else:
# Arbitrary point in outputPivotVal case... If the requested outputPivotVal point Set does not match any of the stored ones and
# start_outputPivotVal <= requested_outputPivotVal_point <= end_outputPivotVal, compute an interpolated value
if allOutParam:
for key in histVar[1]['outputSpaceHeaders']:
if i == 0: outDict[key] = np.zeros(len(histList))
outDict[key][i] = interp1d(
histVar[0][:, pivotIndex],
histVar[0]
[:, histVar[1]['outputSpaceHeaders'].index(key)],
kind='linear')(outputPivotVal)
else:
for key in outParam:
if i == 0: outDict[key] = np.zeros(len(histList))
if key in histVar[1]['outputSpaceHeaders'] or \
utils.toBytes(key) in histVar[1]['outputSpaceHeaders']:
if key in histVar[1]['outputSpaceHeaders']:
outDict[key][i] = interp1d(
histVar[0][:, pivotIndex],
histVar[0][:,
histVar[1]['outputSpaceHeaders']
.index(key)],
kind='linear')(outputPivotVal)
else:
outDict[key][i] = interp1d(
histVar[0][:, pivotIndex],
histVar[0][:,
histVar[1]['outputSpaceHeaders']
.index(utils.toBytes(key))],
kind='linear')(outputPivotVal)
else:
self.raiseAnError(
IOError,
"the parameter " + key + " has not been found")
del histVar
# return tuple of PointSet
return (copy.copy(inDict), copy.copy(outDict), copy.copy(metaDict))
