def printInput(self,outfile=None): """ Method to print out the new input @ In, outfile, string, optional, output file root @ Out, None """ # 4 sub levels maximum def printSubLevels(xmlnode,IOfile,indentMultiplier): IOfile.write(b' '*indentMultiplier+b'[./'+toBytes(xmlnode.tag)+b']\n') for string in xmlnode.tail if xmlnode.tail else []: IOfile.write(b' '*indentMultiplier+string+b'\n') for key in xmlnode.attrib.keys(): IOfile.write(b' '*indentMultiplier+toBytes(key)+b' = '+toBytes(toStrish(xmlnode.attrib[key]))+b'\n') if outfile==None: outfile =self.inputfile IOfile = open(outfile,'wb') for child in self.root: IOfile.write(b'['+toBytes(child.tag)+b']\n') if child.tail: for string in child.tail: IOfile.write(b' '+string+b'\n') for key in child.attrib.keys(): IOfile.write(b' '+toBytes(key)+b' = '+toBytes(toStrish(child.attrib[key]))+b'\n') for childChild in child: printSubLevels(childChild,IOfile,1) for childChildChild in childChild: printSubLevels(childChildChild,IOfile,2) for childChildChildChild in childChildChild: printSubLevels(childChildChildChild,IOfile,3) IOfile.write(b' [../]\n') IOfile.write(b' [../]\n') IOfile.write(b' [../]\n') IOfile.write(b'[]\n')
def printSubLevels(xmlnode, IOfile, indentMultiplier): IOfile.write(b' ' * indentMultiplier + b'[./' + toBytes(xmlnode.tag) + b']\n') for string in xmlnode.tail if xmlnode.tail else []: IOfile.write(b' ' * indentMultiplier + string + b'\n') for key in xmlnode.attrib.keys(): IOfile.write(b' ' * indentMultiplier + toBytes(key) + b' = ' + toBytes(toStrish(xmlnode.attrib[key])) + b'\n')
def __updateGroupLists(self,groupName, parentName): """ Utility method to update the group lists @ In, groupName, str, the new group added @ In, parentName, str, the parent name @ Out, None """ if parentName != "/": self.allGroupPaths.append(utils.toBytes(parentName) + b"/" + utils.toBytes(groupName)) self.allGroupEnds.append(True) else: self.allGroupPaths.append(b"/" + utils.toBytes(groupName)) self.allGroupEnds.append(True)
def __matchDict(self, dictionary, other): """ Method to check the consistency of two dictionaries Returns true if all the keys and values in other match all the keys and values in dictionary. Note that it does not check that all the keys in dictionary match all the keys and values in other. @ In, dictionary, dict, first dictionary to check @ In, other, dict, second dictionary to check @ Out, returnBool, bool, True if all the keys and values in other match all the keys and values in dictionary. """ returnBool = True for key in other: if key in dictionary: #if dictionary[key] != other[key]: if not compare(dictionary[key], other[key]): print("Missmatch ", key, repr(dictionary[key]), repr(other[key])) returnBool = False else: binKey = toBytes(key) if binKey in dictionary: if not compare(dictionary[binKey], other[key]): print("Missmatch_b ", key, dictionary[binKey], other[key]) returnBool = False else: print("No_key ", key, other[key]) returnBool = False return returnBool
def finalizeCodeOutput(self, command, output, workingDir): """ Called by RAVEN to modify output files (if needed) so that they are in a proper form. In this case, OpenModelica CSV output comes with trailing commas that RAVEN doesn't like. So we have to strip them. Also, the first line (with the variable names) has those names enclosed in double quotes (which we have to remove) @ In, command, string, the command used to run the just ended job @ In, output, string, the Output name root @ In, workingDir, string, current working dir @ Out, destFileName, string, present in case the root of the output file gets changed in this method. """ # Make a new temporary file in the working directory and read the lines from the original CSV # to it, stripping trailing commas in the process. tempOutputFD, tempOutputFileName = tempfile.mkstemp(dir = workingDir, text = True) sourceFileName = os.path.join(workingDir, output) # The source file comes in without .csv on it print('sourcefilename:',sourceFileName) destFileName = sourceFileName.replace('rawout~', 'out~') # When fix the CSV, change rawout~ to out~ sourceFileName += '.csv' inputFile = open(sourceFileName) for line in inputFile: # Line ends with a comma followed by a newline #XXX toBytes seems to be needed here in python3, despite the text = True os.write(tempOutputFD, utils.toBytes(line.replace('"','').strip().strip(',') + '\n')) inputFile.close() os.close(tempOutputFD) shutil.move(tempOutputFileName, destFileName + '.csv') return destFileName # Return the name without the .csv on it...RAVEN will add it
def __updateDict(self, dictionary, other): """ Add all the keys and values in other into dictionary @ In, dictionary, dict, dictionary that needs to be updated @ In, other, dict, dictionary from which the valued need to be taken @ Out, None """ for key in other: if key in dictionary: dictionary[key] = other[key] else: binKey = toBytes(key) if binKey in dictionary: dictionary[binKey] = other[key] else: dictionary[key] = other[key]
def __findInXML(self, element, name): """ Checks if there is a tag with name or binary name in element, and returns the (found,actual_name) @ In, element, xml.etree.ElementTree.Element, element where the 'name' needs to be found @ In, name, string, name of the node to be found @ Out, returnTuple, tuple, response. returnTuple[0] bool (True if found) and returnTuple[1] string (binary name in the element) """ returnTuple = None if element.find(name) is not None: returnTuple = (True, name) else: binaryName = toBytes(name) if element.find(binaryName) is not None: returnTuple = (True, binaryName) else: returnTuple = (False, None) return returnTuple
def __isGroup(self,name,obj): """ Function to check if an object name is of type "group". If it is, the function stores its name into the "self.allGroupPaths" list and update the dictionary "self.allGroupEnds" @ In, name, string, object name @ In, obj, object, the object itself @ Out, None """ if isinstance(obj,h5.Group): self.allGroupPaths.append(utils.toBytes(name)) try: self.allGroupEnds.append(obj.attrs["endGroup"]) except KeyError: self.allGroupEnds.append(True) if "rootname" in obj.attrs: self.parentGroupName = name self.raiseAWarning('not found attribute endGroup in group ' + name + '.Set True.') return
def addGroupInit(self, groupName, attributes=None): """ Function to add an empty group to the database This function is generally used when the user provides a rootname in the input. It uses the groupName + it appends the date and time. @ In, groupName, string, group name @ In, attributes, dict, optional, dictionary of attributes that must be added as metadata (None by default) @ Out, None """ attribs = {} if attributes is None else attributes groupNameInit = groupName + "_" + datetime.now().strftime( "%m-%d-%Y-%H-%S") for index in range(len(self.allGroupPaths)): comparisonName = utils.toString(self.allGroupPaths[index]) splittedPath = comparisonName.split('/') if len(splittedPath) > 0: if groupNameInit in splittedPath[0]: alphabetCounter, movingCounter = 0, 0 asciiAlphabet = list(string.ascii_uppercase) prefixLetter = '' while True: testGroup = groupNameInit + "_" + prefixLetter + asciiAlphabet[ alphabetCounter] if testGroup not in self.allGroupPaths: groupNameInit = utils.toString(testGroup) break alphabetCounter += 1 if alphabetCounter >= len(asciiAlphabet): prefix = asciiAlphabet[movingCounter] alphabetCounter = 0 movingCounter += 1 break self.parentGroupName = "/" + groupNameInit # Create the group grp = self.h5FileW.create_group(groupNameInit) # Add metadata grp.attrs.update(attribs) grp.attrs['rootname'] = True grp.attrs['endGroup'] = False grp.attrs[b'groupName'] = groupNameInit self.allGroupPaths.append(utils.toBytes("/" + groupNameInit)) self.allGroupEnds.append(False) self.__updateFileLevelInfoDatasets() self.h5FileW.flush()
def __addSubGroup(self,groupName,attributes,source): """ Function to add a group into the database (Hierarchical) @ In, groupName, string, group name @ In, attributes, dict, dictionary of attributes that must be added as metadata @ In, source, File object, source data @ Out, None """ for index in xrange(len(self.allGroupPaths)): comparisonName = self.allGroupPaths[index] splittedPath=comparisonName.split('/') for splgroup in splittedPath: if groupName == splgroup and splittedPath[0] == self.parentGroupName: self.raiseAnError(IOError,"Group named " + groupName + " already present in database " + self.name + ". new group " + groupName + " is equal to old group " + comparisonName) if source['type'] == 'csv': # Source in CSV format f = open(source['name'],'rb') # Retrieve the headers of the CSV file headers = f.readline().split(b",") # Load the csv into a numpy array(n time steps, n parameters) data = np.loadtxt(f,dtype='float',delimiter=',',ndmin=2) # Check if the parent attribute is not null # In this case append a subgroup to the parent group # Otherwise => it's the main group parentID = None if 'metadata' in attributes.keys(): if 'parentID' in attributes['metadata'].keys(): parentID = attributes['metadata']['parentID'] else: if 'parentID' in attributes.keys(): parentID = attributes['parentID'] if parentID: parentName = parentID else: self.raiseAnError(IOError,'NOT FOUND attribute <parentID> into <attributes> dictionary') # Find parent group path if parentName != '/': parentGroupName = self.__returnParentGroupPath(parentName) else: parentGroupName = parentName # Retrieve the parent group from the HDF5 database if parentGroupName in self.h5FileW: grp = self.h5FileW.require_group(parentGroupName) else: # try to guess the parentID from the file name head,tail = os.path.split(os.path.dirname(source['name'])) testParentName = self.__returnParentGroupPath(tail[:-2]) if testParentName in self.h5FileW: grp = self.h5FileW.require_group(testParentName) else: closestGroup = difflib.get_close_matches(parentName, self.allGroupPaths, n=1, cutoff=0.01) errorString = ' NOT FOUND parent group named "' + str(parentName)+'" for loading file '+str(source['name']) errorString+= '\n Tried '+str(tail[:-2])+ ' but not found as well. All group paths are:\n -'+'\n -'.join(self.allGroupPaths) errorString+= '\n Closest parent group found is "'+str(closestGroup[0] if len(closestGroup) > 0 else 'None')+'"!' self.raiseAnError(ValueError,errorString) # The parent group is not the endgroup for this branch self.allGroupEnds[parentGroupName] = False grp.attrs["EndGroup"] = False self.raiseAMessage('Adding group named "' + groupName + '" in Database "'+ self.name +'"') # Create the sub-group sgrp = grp.create_group(groupName) # Create data set in this new group sgrp.create_dataset(groupName+"_data", dtype="float", data=data) # Add the metadata sgrp.attrs["outputSpaceHeaders" ] = headers sgrp.attrs["nParams" ] = data[0,:].size sgrp.attrs["parent" ] = "root" sgrp.attrs["startTime"] = data[0,0] sgrp.attrs["end_time" ] = data[data[:,0].size-1,0] sgrp.attrs["nTimeSteps" ] = data[:,0].size sgrp.attrs["EndGroup" ] = True sgrp.attrs["sourceType"] = source['type'] if source['type'] == 'csv': sgrp.attrs["sourceFile"] = source['name'] # add metadata if present for attr in attributes.keys(): if attr == 'metadata': if 'SampledVars' in attributes['metadata'].keys(): inpHeaders = [] inpValues = [] for inkey, invalue in attributes['metadata']['SampledVars'].items(): if inkey not in headers: inpHeaders.append(utils.toBytes(inkey)) inpValues.append(invalue) if len(inpHeaders) > 0: sgrp.attrs[b'inputSpaceHeaders'] = inpHeaders sgrp.attrs[b'inputSpaceValues' ] = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(inpValues)))) #Files objects are not JSON serializable, so we have to cover that. #this doesn't cover all possible circumstance, but it covers the DET case. if attr == 'inputFile' and isinstance(attributes[attr][0],Files.File): objectToConvert = list(a.__getstate__() for a in attributes[attr]) else: objectToConvert = mathUtils.convertNumpyToLists(attributes[attr]) converted = json.dumps(objectToConvert) if converted and attr != 'name': sgrp.attrs[utils.toBytes(attr)]=converted else: pass # The sub-group is the new ending group if parentGroupName != "/": self.allGroupPaths.append(parentGroupName + "/" + groupName) self.allGroupEnds[parentGroupName + "/" + groupName] = True else: self.allGroupPaths.append("/" + groupName) self.allGroupEnds["/" + groupName] = True return
def addGroupDataObjects(self,groupName,attributes,source,upGroup=False,specificVars=None): """ Function to add a data (class DataObjects) or Dictionary into the Database @ In, groupName, string, group name @ In, attributes, dict, dictionary of attributes that must be added as metadata @ In, source, dataObject, source data @ In, upGroup, bool, optional, updated group? @ In, specificVars, list(str), if not None then indicates a selective list of variables to include in DB @ Out, None """ if not upGroup: for index in xrange(len(self.allGroupPaths)): comparisonName = self.allGroupPaths[index] splittedPath=comparisonName.split('/') for splgroup in splittedPath: if groupName == splgroup and splittedPath[0] == self.parentGroupName: found = True while found: if groupName in splittedPath: found = True else: found = False groupName = groupName + "_"+ str(index) #self.raiseAnError(IOError,"Group named " + groupName + " already present in database " + self.name + ". new group " + groupName + " is equal to old group " + comparisonName) parentName = self.parentGroupName.replace('/', '') # Create the group if parentName != '/': parentGroupName = self.__returnParentGroupPath(parentName) # Retrieve the parent group from the HDF5 database if parentGroupName in self.h5FileW: parentGroupObj = self.h5FileW.require_group(parentGroupName) else: self.raiseAnError(ValueError,'NOT FOUND group named ' + parentGroupObj) else: parentGroupObj = self.h5FileW if type(source['name']) == dict: # create the group if upGroup: groups = parentGroupObj.require_group(groupName) del groups[groupName+"_data"] else: groups = parentGroupObj.create_group(groupName) groups.attrs[b'mainClass' ] = b'PythonType' groups.attrs[b'sourceType'] = b'Dictionary' # I keep this structure here because I want to maintain the possibility to add a whatever dictionary even if not prepared and divided into output and input sub-sets. A.A. # use ONLY the subset of variables if requested if set(['inputSpaceParams']).issubset(set(source['name'].keys())): sourceInputs = source['name']['inputSpaceParams'].keys() if specificVars is not None: inputHeaders = list(var for var in sourceInputs if var in specificVars) else: inputHeaders = sourceInputs inputHeaders = list(utils.toBytesIterative(inputHeaders)) groups.attrs[b'inputSpaceHeaders' ] = inputHeaders if specificVars is not None: inputValues = list(source['name']['inputSpaceParams'][var] for var in sourceInputs if var in specificVars) else: inputValues = source['name']['inputSpaceParams'].values() inputValues = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(utils.toBytesIterative(inputValues))))) groups.attrs[b'inputSpaceValues' ] = inputValues if set(['outputSpaceParams']).issubset(set(source['name'].keys())): if specificVars is not None: outDict = dict((k,v) for k,v in source['name']['outputSpaceParams'].items() if k in specificVars) else: outDict = source['name']['outputSpaceParams'] else: if specificVars is not None: outDict = dict((key,value) for (key,value) in source['name'].iteritems() if key not in ['inputSpaceParams'] and key in specificVars) else: outDict = dict((key,value) for (key,value) in source['name'].iteritems() if key not in ['inputSpaceParams']) outHeaders = utils.toBytesIterative(list(outDict.keys())) outValues = utils.toBytesIterative(list(outDict.values())) groups.attrs[b'nParams' ] = len(outHeaders) groups.attrs[b'outputSpaceHeaders'] = outHeaders groups.attrs[b'EndGroup' ] = True groups.attrs[b'parentID' ] = parentName maxSize = 0 for value in outValues: if type(value) == np.ndarray or type(value).__name__ == 'c1darray': if maxSize < value.size: actualOne = np.asarray(value).size elif type(value) in [int,float,bool,np.float64,np.float32,np.float16,np.int64,np.int32,np.int16,np.int8,np.bool8]: actualOne = 1 else: self.raiseAnError(IOError,'The type of the dictionary parameters must be within float,bool,int,numpy.ndarray.Got '+type(value).__name__) if maxSize < actualOne: maxSize = actualOne groups.attrs[b'nTimeSteps' ] = maxSize dataout = np.zeros((maxSize,len(outHeaders))) for index in range(len(outHeaders)): if type(outValues[index]) == np.ndarray or type(value).__name__ == 'c1darray': dataout[0:outValues[index].size,index] = np.ravel(outValues[index])[:] else: dataout[0,index] = outValues[index] # create the data set groups.create_dataset(groupName + "_data", dtype="float", data=dataout) # add metadata if present for attr in attributes.keys(): objectToConvert = mathUtils.convertNumpyToLists(attributes[attr]) converted = json.dumps(objectToConvert) if converted and attr != 'name': groups.attrs[utils.toBytes(attr)]=converted if parentGroupName != "/": self.allGroupPaths.append(parentGroupName + "/" + groupName) self.allGroupEnds[parentGroupName + "/" + groupName] = True else: self.allGroupPaths.append("/" + groupName) self.allGroupEnds["/" + groupName] = True else: # Data(structure) # Retrieve the headers from the data (inputs and outputs) inpParams = source['name'].getInpParametersValues().keys() outParams = source['name'].getOutParametersValues().keys() if specificVars is not None: headersIn = list(v for v in inpParams if v in specificVars) headersOut = list(v for v in outParams if v in specificVars) else: headersIn = list(inpParams) headersOut = list(outParams) # for a "HistorySet" type we create a number of groups = number of HistorySet (compatibility with loading structure) if specificVars is not None: dataIn = list(source['name'].getInpParametersValues()[v] for v in inpParams if v in specificVars) dataOut = list(source['name'].getOutParametersValues()[v] for v in outParams if v in specificVars) else: dataIn = list(source['name'].getInpParametersValues().values()) dataOut = list(source['name'].getOutParametersValues().values()) # FIXME unused, but left commented because I'm not sure why they're unused. PT #headersInUnstructured = list(source['name'].getInpParametersValues(self,unstructuredInputs=True).keys()) #dataInUnstructured = list(source['name'].getInpParametersValues(self,unstructuredInputs=True).values()) metadata = source['name'].getAllMetadata() if source['name'].type in ['HistorySet','PointSet']: groups = [] if 'HistorySet' in source['name'].type: nruns = len(dataIn) else: nruns = dataIn[0].size for run in range(nruns): if upGroup: groups.append(parentGroupObj.require_group(groupName + b'|' +str(run))) if (groupName + "_data") in groups[run]: del groups[run][groupName+"_data"] else: groups.append(parentGroupObj.create_group(groupName + '|' +str(run))) groups[run].attrs[b'sourceType'] = utils.toBytes(source['name'].type) groups[run].attrs[b'mainClass' ] = b'DataObjects' groups[run].attrs[b'EndGroup' ] = True groups[run].attrs[b'parentID' ] = parentName if source['name'].type == 'HistorySet': groups[run].attrs[b'inputSpaceHeaders' ] = [utils.toBytes(list(dataIn[run].keys())[i]) for i in range(len(dataIn[run].keys()))] groups[run].attrs[b'outputSpaceHeaders'] = [utils.toBytes(list(dataOut[run].keys())[i]) for i in range(len(dataOut[run].keys()))] json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(dataIn[run].values())))) groups[run].attrs[b'inputSpaceValues' ] = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list(dataIn[run].values())))) groups[run].attrs[b'nParams' ] = len(dataOut[run].keys()) #collect the outputs dataout = np.zeros((next(iter(dataOut[run].values())).size,len(dataOut[run].values()))) for param in range(len(dataOut[run].values())): dataout[:,param] = list(dataOut[run].values())[param][:] groups[run].create_dataset(groupName +'|' +str(run)+"_data" , dtype="float", data=dataout) groups[run].attrs[b'nTimeSteps' ] = next(iter(dataOut[run].values())).size else: groups[run].attrs[b'inputSpaceHeaders' ] = [utils.toBytes(headersIn[i]) for i in range(len(headersIn))] groups[run].attrs[b'outputSpaceHeaders'] = [utils.toBytes(headersOut[i]) for i in range(len(headersOut))] groups[run].attrs[b'inputSpaceValues' ] = json.dumps([list(utils.toListFromNumpyOrC1arrayIterative(np.atleast_1d(np.array(dataIn[x][run])).tolist())) for x in range(len(dataIn))]) groups[run].attrs[b'nParams' ] = len(headersOut) groups[run].attrs[b'nTimeSteps' ] = 1 #collect the outputs dataout = np.zeros((1,len(dataOut))) for param in range(len(dataOut)): dataout[0,param] = dataOut[param][run] groups[run].create_dataset(groupName +'|' +str(run)+"_data", dtype="float", data=dataout) # add metadata if present for attr in attributes.keys(): objectToConvert = mathUtils.convertNumpyToLists(attributes[attr]) converted = json.dumps(objectToConvert) if converted and attr != 'name': groups[run].attrs[utils.toBytes(attr)]=converted for attr in metadata.keys(): if len(metadata[attr]) == nruns: toProcess = metadata[attr][run] else: toProcess = metadata[attr] if type(toProcess).__name__ == 'list' and 'input' in attr.lower() and isinstance(toProcess[0],Files.File): objectToConvert = list(a.__getstate__() for a in toProcess) elif isinstance(toProcess,Files.File): objectToConvert =toProcess.__getstate__() else: objectToConvert = mathUtils.convertNumpyToLists(toProcess) converted = json.dumps(objectToConvert) if converted and attr != 'name': groups[run].attrs[utils.toBytes(attr)]=converted if parentGroupName != "/": self.allGroupPaths.append(parentGroupName + "/" + groupName + '|' +str(run)) self.allGroupEnds[parentGroupName + "/" + groupName + '|' +str(run)] = True else: self.allGroupPaths.append("/" + groupName + '|' +str(run)) self.allGroupEnds["/" + groupName + '|' +str(run)] = True else: self.raiseAnError(IOError,'The function addGroupDataObjects accepts Data(s) or dictionaries as inputs only!!!!!')
def __addGroupRootLevel(self,groupName,attributes,source,upGroup=False): """ Function to add a group into the database (root level) @ In, groupName, string, group name @ In, attributes, dict, dictionary of attributes that must be added as metadata @ In, source, File object, source file @ In, upGroup, bool, optional, updated group? @ Out, None """ # Check in the "self.allGroupPaths" list if a group is already present... # If so, error (Deleting already present information is not desiderable) if not upGroup: for index in xrange(len(self.allGroupPaths)): comparisonName = self.allGroupPaths[index] splittedPath=comparisonName.split('/') for splgroup in splittedPath: if groupName == splgroup and splittedPath[0] == self.parentGroupName: self.raiseAnError(IOError,"Group named " + groupName + " already present in database " + self.name + ". new group " + groupName + " is equal to old group " + comparisonName) if source['type'] == 'csv': # Source in CSV format f = open(source['name'],'rb') # Retrieve the headers of the CSV file firstRow = f.readline().strip(b"\r\n") #firstRow = f.readline().translate(None,"\r\n") headers = firstRow.split(b",") # if there is the alias system, replace the variable name if 'alias' in attributes.keys(): for aliasType in attributes['alias'].keys(): for var in attributes['alias'][aliasType].keys(): if attributes['alias'][aliasType][var].strip() in headers: headers[headers.index(attributes['alias'][aliasType][var].strip())] = var.strip() else: metadataPresent = True if 'metadata' in attributes.keys() and 'SampledVars' in attributes['metadata'].keys() else False if not (metadataPresent and var.strip() in attributes['metadata']['SampledVars'].keys()): self.raiseAWarning('the ' + aliasType +' alias"'+var.strip()+'" has been defined but has not been found among the variables!') # Load the csv into a numpy array(n time steps, n parameters) data = np.loadtxt(f,dtype='float',delimiter=',',ndmin=2) # First parent group is the root name parentName = self.parentGroupName.replace('/', '') # Create the group if parentName != '/': parentGroupName = self.__returnParentGroupPath(parentName) # Retrieve the parent group from the HDF5 database if parentGroupName in self.h5FileW: rootgrp = self.h5FileW.require_group(parentGroupName) else: self.raiseAnError(ValueError,'NOT FOUND group named "' + parentGroupName+'" for loading file '+str(source['name'])) if upGroup: grp = rootgrp.require_group(groupName) del grp[groupName+"_data"] else: grp = rootgrp.create_group(groupName) else: if upGroup: grp = self.h5FileW.require_group(groupName) else: grp = self.h5FileW.create_group(groupName) self.raiseAMessage('Adding group named "' + groupName + '" in DataBase "'+ self.name +'"') # Create dataset in this newly added group grp.create_dataset(groupName+"_data", dtype="float", data=data) # Add metadata grp.attrs["outputSpaceHeaders" ] = headers grp.attrs["nParams" ] = data[0,:].size grp.attrs["parentID" ] = "root" grp.attrs["startTime" ] = data[0,0] grp.attrs["end_time" ] = data[data[:,0].size-1,0] grp.attrs["nTimeSteps" ] = data[:,0].size grp.attrs["EndGroup" ] = True grp.attrs["sourceType" ] = source['type'] if source['type'] == 'csv': grp.attrs["sourceFile"] = source['name'] for attr in attributes.keys(): if attr == 'metadata': if 'SampledVars' in attributes['metadata'].keys(): inpHeaders = [] inpValues = [] for inkey, invalue in attributes['metadata']['SampledVars'].items(): if inkey not in headers: inpHeaders.append(utils.toBytes(inkey)) inpValues.append(invalue) if len(inpHeaders) > 0: grp.attrs[b'inputSpaceHeaders'] = inpHeaders grp.attrs[b'inputSpaceValues' ] = json.dumps(list(utils.toListFromNumpyOrC1arrayIterative(list( inpValues)))) objectToConvert = mathUtils.convertNumpyToLists(attributes[attr]) for o,obj in enumerate(objectToConvert): if isinstance(obj,Files.File): objectToConvert[o]=obj.getFilename() converted = json.dumps(objectToConvert) if converted and attr != 'name': grp.attrs[utils.toBytes(attr)]=converted #decoded = json.loads(grp.attrs[utils.toBytes(attr)]) if "inputFile" in attributes.keys(): grp.attrs[utils.toString("inputFile")] = utils.toString(" ".join(attributes["inputFile"])) if type(attributes["inputFile"]) == type([]) else utils.toString(attributes["inputFile"]) else: self.raiseAnError(ValueError,source['type'] + " unknown!") # Add the group name into the list "self.allGroupPaths" and # set the relative bool flag into the dictionary "self.allGroupEnds" if parentGroupName != "/": self.allGroupPaths.append(parentGroupName + "/" + groupName) self.allGroupEnds[parentGroupName + "/" + groupName] = True else: self.allGroupPaths.append("/" + groupName) self.allGroupEnds["/" + groupName] = True
def __retrieveDataPointSet(self, attributes): """ Function to retrieve a PointSet from the HDF5 database @ In, attributes, dict, options (metadata must be appended to the root group) @ Out, tupleVar, tuple, tuple in which the first position is a dictionary of numpy arays (input variable) and the second is a dictionary of the numpy arrays (output variables). """ # Check the outParam variables and the outputPivotVal filters inParam, outParam, inputRow, outputRow = attributes[ 'inParam'], attributes['outParam'], copy.deepcopy( attributes.get('inputRow', None)), copy.deepcopy( attributes.get('outputRow', None)) inputPivotVal, outputPivotVal, operator = attributes.get( 'inputPivotValue', None), attributes.get('outputPivotValue', None), attributes.get('operator', None) pivotParameter = attributes.get('pivotParameter', None) if outParam == 'all': allOutParam = True else: allOutParam = False if outputPivotVal != None: if 'end' in outputPivotVal: outputPivotValEnd = True else: outputPivotValEnd, outputPivotVal = False, float( outputPivotVal) else: if operator is None: outputPivotValEnd = True else: outputPivotValEnd = False if inputRow == None and inputPivotVal == None: inputRow = 0 if inputRow == None and inputPivotVal == None: inputRow = 0 if inputRow != None: inputRow = int(inputRow) if inputRow > 0: inputRow -= 1 if outputRow != None: outputRow = int(outputRow) if outputRow > 0: outputRow -= 1 inDict = {} outDict = {} metaDict = {} histList = attributes['HistorySet'] # Retrieve all the associated HistorySet and process them for i in range(len(histList)): # Load the data into the numpy array attributes['history'] = histList[i] histVar = self.returnHistory(attributes) #look for pivotParameter if pivotParameter != None: pivotIndex = histVar[1]['outputSpaceHeaders'].index( pivotParameter) if pivotParameter in histVar[1][ 'outputSpaceHeaders'] else None if pivotIndex == None: self.raiseAnError( IOError, 'pivotParameter ' + pivotParameter + ' has not been found in history ' + str(attributes['history']) + '!') else: pivotIndex = histVar[1]['outputSpaceHeaders'].index( "time" ) if "time" in histVar[1]['outputSpaceHeaders'] else None # if None...default is 0 if pivotIndex == None: pivotIndex = 0 if inputRow > histVar[0][:, 0].size - 1 and inputRow != -1: self.raiseAnError( IOError, 'inputRow is greater than number of actual rows in history ' + str(attributes['history']) + '!') # check metadata if 'metadata' in histVar[1].keys(): metaDict[i] = histVar[1]['metadata'] else: metaDict[i] = None for key in inParam: if 'inputSpaceHeaders' in histVar[1]: inInKey = utils.keyIn(histVar[1]['inputSpaceHeaders'], key) inOutKey = utils.keyIn(histVar[1]['outputSpaceHeaders'], key) if inInKey != None: ix = histVar[1]['inputSpaceHeaders'].index(inInKey) if i == 0: inDict[key] = np.zeros(len(histList)) inDict[key][i] = np.atleast_1d( histVar[1]['inputSpaceValues'][ix])[0] elif inOutKey != None and inInKey == None: ix = histVar[1]['outputSpaceHeaders'].index(inOutKey) if i == 0: inDict[key] = np.zeros(len(histList)) if inputPivotVal != None: if float(inputPivotVal) > np.max( histVar[0][:, pivotIndex]) or float( inputPivotVal) < np.min( histVar[0][:, pivotIndex]): self.raiseAnError( IOError, 'inputPivotVal is out of the min and max for input ' + key + ' in Database ' + str(self.name) + '!') inDict[key][i] = interp1d( histVar[0][:, pivotIndex], histVar[0][:, ix], kind='linear')(float(inputPivotVal)) else: inDict[key][i] = histVar[0][inputRow, ix] else: self.raiseAnError( IOError, 'the parameter ' + key + ' has not been found') else: inKey = utils.keyIn(histVar[1]['outputSpaceHeaders'], key) if inKey is not None: ix = histVar[1]['outputSpaceHeaders'].index(inKey) if i == 0: inDict[key] = np.zeros(len(histList)) if inputPivotVal != None: if float(inputPivotVal) > np.max( histVar[0][:, pivotIndex]) or float( inputPivotVal) < np.min( histVar[0][:, pivotIndex]): self.raiseAnError( IOError, 'inputPivotVal is out of the min and max for input ' + key + ' in Database ' + str(self.name) + '!') inDict[key][i] = interp1d( histVar[0][:, pivotIndex], histVar[0][:, ix], kind='linear')(float(inputPivotVal)) else: inDict[key][i] = histVar[0][inputRow, ix] else: self.raiseAnError( IOError, 'the parameter ' + key + ' has not been found in ' + str(histVar[1])) # outputPivotVal end case => PointSet is at the final status if outputPivotValEnd: if allOutParam: for key in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) outDict[key][i] = histVar[0][ -1, histVar[1]['outputSpaceHeaders'].index(key)] else: for key in outParam: if key in histVar[1]['outputSpaceHeaders'] or \ utils.toBytes(key) in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) if key in histVar[1]['outputSpaceHeaders']: outDict[key][i] = histVar[0][ -1, histVar[1]['outputSpaceHeaders']. index(key)] else: outDict[key][i] = histVar[0][ -1, histVar[1]['outputSpaceHeaders']. index(utils.toBytes(key))] else: self.raiseAnError( IOError, "the parameter " + str(key) + " has not been found") elif outputRow != None: if outputRow > histVar[0][:, 0].size - 1 and outputRow != -1: self.raiseAnError( IOError, 'outputRow is greater than number of actual rows in Database ' + str(self.name) + '!') if allOutParam: for key in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) outDict[key][i] = histVar[0][ outputRow, histVar[1]['outputSpaceHeaders'].index(key)] else: for key in outParam: if key in histVar[1]['outputSpaceHeaders'] or \ utils.toBytes(key) in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) if key in histVar[1]['outputSpaceHeaders']: outDict[key][i] = histVar[0][ outputRow, histVar[1]['outputSpaceHeaders'].index(key )] else: outDict[key][i] = histVar[0][ outputRow, histVar[1]['outputSpaceHeaders']. index(utils.toBytes(key))] else: self.raiseAnError( IOError, "the parameter " + str(key) + " has not been found") elif operator != None: if operator not in ['max', 'min', 'average']: self.raiseAnError( IOError, 'operator unknown. Available are min,max,average') if histVar[1]['outputSpaceHeaders']: for key in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) if operator == 'max': outDict[key][i] = np.max( histVar[0] [:, histVar[1]['outputSpaceHeaders'].index(key)]) if operator == 'min': outDict[key][i] = np.min( histVar[0] [:, histVar[1]['outputSpaceHeaders'].index(key)]) if operator == 'average': outDict[key][i] = np.average( histVar[0] [:, histVar[1]['outputSpaceHeaders'].index(key)]) else: for key in outParam: if key in histVar[1]['outputSpaceHeaders'] or \ utils.toBytes(key) in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) if key in histVar[1]['outputSpaceHeaders']: if operator == 'max': outDict[key][i] = np.max( histVar[0] [:, histVar[1]['outputSpaceHeaders']. index(key)]) if operator == 'min': outDict[key][i] = np.min( histVar[0] [:, histVar[1]['outputSpaceHeaders']. index(key)]) if operator == 'average': outDict[key][i] = np.average( histVar[0] [:, histVar[1]['outputSpaceHeaders']. index(key)]) else: if operator == 'max': outDict[key][i] = np.max( histVar[0] [:, histVar[1]['outputSpaceHeaders']. index(utils.toBytes(key))]) if operator == 'min': outDict[key][i] = np.min( histVar[0] [:, histVar[1]['outputSpaceHeaders']. index(utils.toBytes(key))]) if operator == 'average': outDict[key][i] = np.average( histVar[0] [:, histVar[1]['outputSpaceHeaders']. index(utils.toBytes(key))]) else: self.raiseAnError( IOError, "the parameter " + str(key) + " has not been found") else: # Arbitrary point in outputPivotVal case... If the requested outputPivotVal point Set does not match any of the stored ones and # start_outputPivotVal <= requested_outputPivotVal_point <= end_outputPivotVal, compute an interpolated value if allOutParam: for key in histVar[1]['outputSpaceHeaders']: if i == 0: outDict[key] = np.zeros(len(histList)) outDict[key][i] = interp1d( histVar[0][:, pivotIndex], histVar[0] [:, histVar[1]['outputSpaceHeaders'].index(key)], kind='linear')(outputPivotVal) else: for key in outParam: if i == 0: outDict[key] = np.zeros(len(histList)) if key in histVar[1]['outputSpaceHeaders'] or \ utils.toBytes(key) in histVar[1]['outputSpaceHeaders']: if key in histVar[1]['outputSpaceHeaders']: outDict[key][i] = interp1d( histVar[0][:, pivotIndex], histVar[0][:, histVar[1]['outputSpaceHeaders'] .index(key)], kind='linear')(outputPivotVal) else: outDict[key][i] = interp1d( histVar[0][:, pivotIndex], histVar[0][:, histVar[1]['outputSpaceHeaders'] .index(utils.toBytes(key))], kind='linear')(outputPivotVal) else: self.raiseAnError( IOError, "the parameter " + key + " has not been found") del histVar # return tuple of PointSet return (copy.copy(inDict), copy.copy(outDict), copy.copy(metaDict))