def _specializedLoadXMLandCSV(self, filenameRoot, options): """ Function to load the xml additional file of the csv for data (it contains metadata, etc). It must be implemented by the specialized classes @ In, filenameRoot, string, file name root @ In, options, dict, dictionary -> options for loading @ Out, None """ #For Pointset it will create an XML file and one CSV file. #The CSV file will have a header with the input names and output #names, and multiple lines of data with the input and output #numeric values, one line for each input. if options is not None and 'fileToLoad' in options.keys(): name = os.path.join(options['fileToLoad'].getPath(), options['fileToLoad'].getBase()) else: name = self.name filenameLocal = os.path.join(filenameRoot, name) if os.path.isfile(filenameLocal + '.xml'): xmlData = self._loadXMLFile(filenameLocal) assert (xmlData["fileType"] == "Pointset") if "metadata" in xmlData: self._dataContainer['metadata'] = xmlData["metadata"] mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"]) else: mainCSV = os.path.join(filenameRoot, name + '.csv') myFile = open(mainCSV, "rU") header = myFile.readline().rstrip() inoutKeys = header.split(",") inoutValues = [[] for _ in range(len(inoutKeys))] for line in myFile.readlines(): lineList = line.rstrip().split(",") for i in range(len(inoutKeys)): inoutValues[i].append(utils.partialEval(lineList[i])) #extend the expected size of this point set self.numAdditionalLoadPoints = len( inoutValues[0]) #used in checkConsistency self._dataContainer['inputs'] = {} self._dataContainer['outputs'] = {} inoutDict = {} for key, value in zip(inoutKeys, inoutValues): inoutDict[key] = value for key in self.getParaKeys('inputs'): self._dataContainer["inputs"][key] = c1darray( values=np.array(inoutDict[key])) for key in self.getParaKeys('outputs'): self._dataContainer["outputs"][key] = c1darray( values=np.array(inoutDict[key]))
def _updateSpecializedMetadata(self, name, value, options=None): """ This function performs the updating of the values (metadata) into this Data @ In, name, string, parameter name (ex. probability) @ In, value, whatever type, newer value @ Out, None NB. This method, if the metadata name is already present, replaces it with the new value. No appending here, since the metadata are dishomogenius and a common updating strategy is not feasable. """ valueType = None if utils.checkTypeRecursively(value) not in [ 'str', 'unicode', 'bytes' ] else object if options and self._dataParameters['hierarchical']: # we retrieve the node in which the specialized 'Point' has been stored parentID = None if 'metadata' in options.keys(): prefix = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: prefix = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] if parentID: tsnode = self.retrieveNodeInTreeMode(prefix, parentID) self._dataContainer = tsnode.get('dataContainer') if not self._dataContainer: tsnode.add('dataContainer', {'metadata': {}}) self._dataContainer = tsnode.get('dataContainer') else: if 'metadata' not in self._dataContainer.keys(): self._dataContainer['metadata'] = {} if name in self._dataContainer['metadata'].keys(): self._dataContainer['metadata'][name].append( np.atleast_1d(value) ) # = np.concatenate((self._dataContainer['metadata'][name],np.atleast_1d(value))) else: valueToAdd = np.array( value, dtype=valueType ) if valueType is not None else np.array(value) self._dataContainer['metadata'][name] = c1darray( values=np.atleast_1d(valueToAdd)) self.addNodeInTreeMode(tsnode, options) else: if name in self._dataContainer['metadata'].keys(): self._dataContainer['metadata'][name].append( np.atleast_1d(value) ) # = np.concatenate((self._dataContainer['metadata'][name],np.atleast_1d(value))) else: valueToAdd = np.array( value, dtype=valueType ) if valueType is not None else np.array(value) self._dataContainer['metadata'][name] = c1darray( values=np.atleast_1d(valueToAdd))
def _updateSpecializedOutputValue(self, name, value, options=None): """ This function performs the updating of the values (output space) into this Data @ In, name, string, parameter name (ex. cladTemperature) @ In, value, float, newer value (single value) @ Out, None """ if options and self._dataParameters['hierarchical']: # we retrieve the node in which the specialized 'Point' has been stored parentID = None if 'metadata' in options.keys(): prefix = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: prefix = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] if parentID: tsnode = self.retrieveNodeInTreeMode(prefix, parentID) #if 'parentID' in options.keys(): tsnode = self.retrieveNodeInTreeMode(options['prefix'], options['parentID']) #else: tsnode = self.retrieveNodeInTreeMode(options['prefix']) # we store the pointer to the container in the self._dataContainer because checkConsistency acts on this self._dataContainer = tsnode.get('dataContainer') if not self._dataContainer: tsnode.add('dataContainer', {'inputs': {}, 'outputs': {}}) self._dataContainer = tsnode.get('dataContainer') if name in self._dataContainer['outputs'].keys(): self._dataContainer['outputs'].pop(name) if name not in self._dataParameters['outParam']: self._dataParameters['outParam'].append(name) self._dataContainer['outputs'][name] = c1darray( values=np.atleast_1d( value)) #np.atleast_1d(np.atleast_1d(value)[-1]) self.addNodeInTreeMode(tsnode, options) else: if name in self._dataContainer['outputs'].keys(): #popped = self._dataContainer['outputs'].pop(name) self._dataContainer['outputs'][name].append( np.atleast_1d(value)[-1] ) #= copy.copy(np.concatenate((np.array(popped), np.atleast_1d(np.atleast_1d(value)[-1])))) else: if name not in self._dataParameters['outParam']: self._dataParameters['outParam'].append(name) self._dataContainer['outputs'][name] = c1darray( values=np.atleast_1d(np.atleast_1d( value)[-1])) # np.atleast_1d(np.atleast_1d(value)[-1])
def _specializedLoadXMLandCSV(self, filenameRoot, options): """ Function to load the xml additional file of the csv for data (it contains metadata, etc). It must be implemented by the specialized classes @ In, filenameRoot, string, file name root @ In, options, dict, dictionary -> options for loading @ Out, None """ #For Pointset it will create an XML file and one CSV file. #The CSV file will have a header with the input names and output #names, and multiple lines of data with the input and output #numeric values, one line for each input. if options is not None and 'fileToLoad' in options.keys(): name = os.path.join(options['fileToLoad'].getPath(), options['fileToLoad'].getBase()) else: name = self.name filenameLocal = os.path.join(filenameRoot, name) if os.path.isfile(filenameLocal + '.xml'): xmlData = self._loadXMLFile(filenameLocal) assert (xmlData["fileType"] == "Pointset") if "metadata" in xmlData: self._dataContainer['metadata'] = xmlData["metadata"] mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"]) else: mainCSV = os.path.join(filenameRoot, name + '.csv') myFile = open(mainCSV, "rU") header = myFile.readline().rstrip() inoutKeys = header.split(",") inoutValues = [[] for _ in range(len(inoutKeys))] for lineNumber, line in enumerate(myFile.readlines(), 2): lineList = line.rstrip().split(",") for i in range(len(inoutKeys)): datum = utils.partialEval(lineList[i]) if datum == '': self.raiseAnError( IOError, 'Invalid data in input file: {} at line {}: "{}"'. format(filenameLocal, lineNumber, line.rstrip())) inoutValues[i].append(utils.partialEval(lineList[i])) #extend the expected size of this point set self.numAdditionalLoadPoints += len( inoutValues[0]) #used in checkConsistency ## Do not reset these containers because it will wipe whatever information ## already exists in this PointSet. This is not one of the use cases for our ## data objects. We claim in the manual to construct or update information. ## These should be non-destructive operations. -- DPM 6/26/17 # self._dataContainer['inputs'] = {} # self._dataContainer['outputs'] = {} inoutDict = {} for key, value in zip(inoutKeys, inoutValues): inoutDict[key] = value for key in self.getParaKeys('inputs'): ## Again, in order to be non-destructive we should only initialize on the ## first go-around, subsequent loads should append to the existing list. ## -- DPM 6/26/17 if key not in self._dataContainer["inputs"]: self._dataContainer["inputs"][key] = c1darray( values=np.array(inoutDict[key])) else: self._dataContainer["inputs"][key].append( c1darray(values=np.array(inoutDict[key]))) for key in self.getParaKeys('outputs'): ## Again, in order to be non-destructive we should only initialize on the ## first go-around, subsequent loads should append to the existing list. ## -- DPM 6/26/17 if key not in self._dataContainer["outputs"]: self._dataContainer["outputs"][key] = c1darray( values=np.array(inoutDict[key])) else: self._dataContainer["outputs"][key].append( c1darray(values=np.array(inoutDict[key])))
def _updateSpecializedOutputValue(self, name, value, options=None): """ This function performs the updating of the values (output space) into this Data @ In, name, string, parameter name (ex. cladTemperature) @ In, value, float, newer value (single value) @ Out, None """ ## So, you are trying to update a single data point, but you passed in ## more information, this means we need to reduce it down using one of our ## recipes. value = np.atleast_1d(value).flatten() if len(value) > 1: if options is None: outputRow = -1 outputPivotVal = None operator = None else: ## Not sure if any of these are necessary, but I am trying to replicate ## the magic that takes place in the CsvLoader -- DPM 5/3/2017 outputRow = copy.deepcopy(options.get('outputRow', -1)) outputPivotVal = options.get('outputPivotValue', None) operator = options.get('operator', None) if outputRow is None: if outputPivotVal is not None and 'end' in outputPivotVal: outputRow = -1 # elif outputPivotVal != None: # outputPivotVal = float(outputPivotVal) if operator == 'max': value = np.max(value) elif operator == 'min': value = np.min(value) elif operator == 'average': value = np.average(value) else: #elif outputRow is not None: value = value[outputRow] ## We don't have access to the pivot parameter's information at this ## point, so I will forego this implementation for now -- DPM 5/3/2017 #else: # value = interp1d(data[:,pivotIndex], value, kind='linear')(outputPivotVal) if options and self._dataParameters['hierarchical']: # we retrieve the node in which the specialized 'Point' has been stored parentID = None if 'metadata' in options.keys(): prefix = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: prefix = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] if parentID: tsnode = self.retrieveNodeInTreeMode(prefix, parentID) #if 'parentID' in options.keys(): tsnode = self.retrieveNodeInTreeMode(options['prefix'], options['parentID']) #else: tsnode = self.retrieveNodeInTreeMode(options['prefix']) # we store the pointer to the container in the self._dataContainer because checkConsistency acts on this self._dataContainer = tsnode.get('dataContainer') if not self._dataContainer: tsnode.add('dataContainer', {'inputs': {}, 'outputs': {}}) self._dataContainer = tsnode.get('dataContainer') if name in self._dataContainer['outputs'].keys(): self._dataContainer['outputs'].pop(name) if name not in self._dataParameters['outParam']: self._dataParameters['outParam'].append(name) self._dataContainer['outputs'][name] = c1darray( values=np.atleast_1d( value)) #np.atleast_1d(np.atleast_1d(value)[-1]) self.addNodeInTreeMode(tsnode, options) else: if name in self._dataContainer['outputs'].keys(): #popped = self._dataContainer['outputs'].pop(name) self._dataContainer['outputs'][name].append( np.atleast_1d(value)[-1] ) #= copy.copy(np.concatenate((np.array(popped), np.atleast_1d(np.atleast_1d(value)[-1])))) else: if name not in self._dataParameters['outParam']: self._dataParameters['outParam'].append(name) self._dataContainer['outputs'][name] = c1darray( values=np.atleast_1d(np.atleast_1d( value)[-1])) # np.atleast_1d(np.atleast_1d(value)[-1])
def _updateSpecializedInputValue(self, name, value, options=None): """ This function performs the updating of the values (input space) into this Data @ In, name, string, parameter name (ex. cladTemperature) @ In, value, float, newer value (single value) @ Out, None """ # if this flag is true, we accept realizations in the input space that are not only scalar but can be 1-D arrays! #acceptArrayRealizations = False if options == None else options.get('acceptArrayRealizations',False) unstructuredInput = False if isinstance(value, (np.ndarray, c1darray)): if np.asarray(value).ndim > 1 and max( np.asarray(value).shape) != np.asarray(value).size: self.raiseAnError( NotConsistentData, 'PointSet Data accepts only a 1 Dimensional numpy array or a single value for method <_updateSpecializedInputValue>. Array shape is ' + str(value.shape)) #if value.size != 1 and not acceptArrayRealizations: self.raiseAnError(NotConsistentData,'PointSet Data accepts only a numpy array of dim 1 or a single value for method <_updateSpecializedInputValue>. Size is ' + str(value.size)) unstructuredInput = True if value.size > 1 else False if options and self._dataParameters['hierarchical']: # we retrieve the node in which the specialized 'Point' has been stored parentID = None if 'metadata' in options.keys(): prefix = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: prefix = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] if parentID: tsnode = self.retrieveNodeInTreeMode(prefix, parentID) else: tsnode = self.retrieveNodeInTreeMode(prefix) self._dataContainer = tsnode.get('dataContainer') if not self._dataContainer: tsnode.add('dataContainer', { 'inputs': {}, 'unstructuredInputs': {}, 'outputs': {} }) self._dataContainer = tsnode.get('dataContainer') if name in self._dataContainer['inputs'].keys(): self._dataContainer['inputs'].pop(name) if name in self._dataContainer['unstructuredInputs'].keys(): self._dataContainer['unstructuredInputs'].pop(name) if name not in self._dataParameters['inParam']: self._dataParameters['inParam'].append(name) if not unstructuredInput: self._dataContainer['inputs'][name] = c1darray( values=np.atleast_1d(np.ravel(value)[-1])) else: self._dataContainer['unstructuredInputs'][name] = [ c1darray(values=np.atleast_1d(np.ravel(value))) ] #self._dataContainer['inputs'][name] = c1darray(values=np.atleast_1d(np.atleast_1d(value)[-1])) if not acceptArrayRealizations else c1darray(values=np.atleast_1d(np.atleast_1d(value))) self.addNodeInTreeMode(tsnode, options) else: if name in itertools.chain( self._dataContainer['inputs'].keys(), self._dataContainer['unstructuredInputs'].keys()): #popped = self._dataContainer['inputs'].pop(name) if not unstructuredInput: self._dataContainer['inputs'][name].append( np.atleast_1d(np.ravel(value)[-1])) else: self._dataContainer['unstructuredInputs'][name].append( np.atleast_1d(np.ravel(value))) #self._dataContainer['inputs'][name] = c1darray(values=np.atleast_1d(np.atleast_1d(value)[-1])) copy.copy(np.concatenate((np.atleast_1d(np.array(popped)), np.atleast_1d(np.atleast_1d(value)[-1])))) else: if name not in self._dataParameters['inParam']: self._dataParameters['inParam'].append(name) #if name not in self._dataParameters['inParam']: self.raiseAnError(NotConsistentData,'The input variable '+name+'is not among the input space of the DataObject '+self.name) #self._dataContainer['inputs'][name] = c1darray(values=np.atleast_1d(np.atleast_1d(value)[-1])) if not acceptArrayRealizations else c1darray(values=np.atleast_1d(np.atleast_1d(value))) if not unstructuredInput: self._dataContainer['inputs'][name] = c1darray( values=np.atleast_1d(np.ravel(value)[-1])) else: self._dataContainer['unstructuredInputs'][name] = [ c1darray(values=np.atleast_1d(np.ravel(value))) ]
def _specializedLoadXMLandCSV(self, filenameRoot, options): """ Function to load the xml additional file of the csv for data (it contains metadata, etc). It must be implemented by the specialized classes @ In, filenameRoot, string, file name root @ In, options, dict, dictionary -> options for loading @ Out, None """ #For HistorySet, create an XML file, and multiple CSV #files. The first CSV file has a header with the input names, #and a column for the filenames. There is one CSV file for each #data line in the first CSV and they are named with the #filename. They have the output names for a header, a column #for time, and the rest of the file is data for different times. if options is not None and 'fileToLoad' in options.keys(): name = os.path.join(options['fileToLoad'].getPath(), options['fileToLoad'].getBase()) else: name = self.name filenameLocal = os.path.join(filenameRoot, name) if os.path.isfile(filenameLocal + '.xml'): xmlData = self._loadXMLFile(filenameLocal) assert (xmlData["fileType"] == "HistorySet") if "metadata" in xmlData: self._dataContainer['metadata'] = xmlData["metadata"] mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"]) else: mainCSV = os.path.join(filenameRoot, name + '.csv') myFile = open(mainCSV, "rU") header = myFile.readline().rstrip() inpKeys = header.split(",")[:-1] inpValues = [] outKeys = [] outValues = [] allLines = myFile.readlines() for mainLine in allLines: mainLineList = mainLine.rstrip().split(",") inpValues_h = [utils.partialEval(a) for a in mainLineList[:-1]] inpValues.append(inpValues_h) dataFilename = mainLineList[-1] subCSVFilename = os.path.join(filenameRoot, dataFilename) subCSVFile = Files.returnInstance("CSV", self) subCSVFile.setFilename(subCSVFilename) self._toLoadFromList.append(subCSVFile) with open(subCSVFilename, "rU") as myDataFile: header = myDataFile.readline().rstrip() outKeys_h = header.split(",") outValues_h = [[] for a in range(len(outKeys_h))] for lineNumber, line in enumerate(myDataFile.readlines(), 2): lineList = line.rstrip().split(",") for i in range(len(outKeys_h)): datum = utils.partialEval(lineList[i]) if datum == '': self.raiseAnError( IOError, 'Invalid data in input file: {} at line {}: "{}"' .format(subCSVFilename, lineNumber, line.rstrip())) outValues_h[i].append(datum) myDataFile.close() outKeys.append(outKeys_h) outValues.append(outValues_h) ## Do not reset these containers because it will wipe whatever information ## already exists in this HistorySet. This is not one of the use cases for ## our data objects. We claim in the manual to construct or update ## information. These should be non-destructive operations. -- DPM 6/26/17 # self._dataContainer['inputs'] = {} #XXX these are indexed by 1,2,... # self._dataContainer['outputs'] = {} #XXX these are indexed by 1,2,... startKey = len(self._dataContainer['inputs'].keys()) for i in range(len(inpValues)): mainKey = startKey + i + 1 subInput = {} subOutput = {} for key, value in zip(inpKeys, inpValues[i]): #subInput[key] = c1darray(values=np.array([value]*len(outValues[0][0]))) if key in self.getParaKeys('inputs'): subInput[key] = c1darray(values=np.array([value])) for key, value in zip(outKeys[i], outValues[i]): if key in self.getParaKeys('outputs'): subOutput[key] = c1darray(values=np.array(value)) self._dataContainer['inputs'][mainKey] = subInput self._dataContainer['outputs'][mainKey] = subOutput #extend the expected size of this point set self.numAdditionalLoadPoints += len( allLines) #used in checkConsistency self.checkConsistency()
def _updateSpecializedOutputValue(self, name, value, options=None): """ This function performs the updating of the values (output space) into this Data @ In, name, either 1) list (size = 2), name[0] == history number(ex. 1 or 2 etc) - name[1], parameter name (ex. cladTemperature) or 2) string, parameter name (ex. cladTemperature) -> in this second case,the parameter is added in the last history (if not present), otherwise a new history is created and the new value is inserted in it @ In, value, ?, ? @ Out, None """ if isinstance(value, np.ndarray): #self.raiseADebug('FIXME: Converted np.ndarray into c1darray in HistorySet!') value = c1darray(values=value) if not isinstance(value, c1darray): self.raiseAnError( NotConsistentData, 'HistorySet Data accepts only cached_ndarray as type for method <_updateSpecializedOutputValue>. Got ' + str(type(value))) if options and self._dataParameters['hierarchical']: parentID = None if type(name) == list: namep = name[1] if type(name[0]) == str: nodeId = name[0] else: if 'metadata' in options.keys(): nodeId = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: nodeId = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] else: if 'metadata' in options.keys(): nodeId = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: nodeId = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] namep = name if parentID: tsnode = self.retrieveNodeInTreeMode(nodeId, parentID) # we store the pointer to the container in the self._dataContainer because checkConsistency acts on this self._dataContainer = tsnode.get('dataContainer') if not self._dataContainer: tsnode.add('dataContainer', {'inputs': {}, 'outputs': {}}) self._dataContainer = tsnode.get('dataContainer') if namep in self._dataContainer['outputs'].keys(): self._dataContainer['outputs'].pop(namep) if namep not in self._dataParameters['outParam']: self._dataParameters['outParam'].append(namep) self._dataContainer['outputs'][namep] = c1darray( values=np.atleast_1d(np.array(value, dtype=float))) self.addNodeInTreeMode(tsnode, options) else: resultsArray = c1darray( values=np.atleast_1d(np.array(value, dtype=float))) if type(name) == list: # there are info regarding the history number try: self._dataContainer['outputs'][name[0]][ name[1]] = resultsArray except KeyError: self._dataContainer['outputs'][name[0]] = { name[1]: resultsArray } else: # no info regarding the history number => use internal counter if len(self._dataContainer['outputs']) == 0: self._dataContainer['outputs'][1] = {name: resultsArray} else: hisn = max(self._dataContainer['outputs'].keys()) if name in list( self._dataContainer['outputs'].values())[-1]: hisn += 1 self._dataContainer['outputs'][hisn] = {} self._dataContainer['outputs'][hisn][name] = copy.copy( resultsArray ) #FIXME why deepcopy here but not elsewhere?
def _updateSpecializedInputValue(self, name, value, options=None): """ This function performs the updating of the values (input space) into this Data @ In, name, either 1) list (size = 2), name[0] == history number(ex. 1 or 2 etc) - name[1], parameter name (ex. cladTemperature) or 2) string, parameter name (ex. cladTemperature) -> in this second case,the parameter is added in the last history (if not present), otherwise a new history is created and the new value is inserted in it @ In, value, newer value @ Out, None """ # if this flag is true, we accept realizations in the input space that are not only scalar but can be 1-D arrays! #acceptArrayRealizations = False if options == None else options.get('acceptArrayRealizations',False) unstructuredInput = False if isinstance(value, np.ndarray): if value.shape == ( ): #can't cast single-entry ND array into a c1darray, so make it into a single entry value = value.dtype.type(value) else: value = c1darray(values=value) if not isinstance(value, (float, int, bool, c1darray)): self.raiseAnError( NotConsistentData, 'HistorySet Data accepts only a numpy array or a single value for method <_updateSpecializedInputValue>. Got type ' + str(type(value))) if isinstance(value, c1darray): if np.asarray(value).ndim > 1 and max( value.values.shape) != np.asarray(value).size: self.raiseAnError( NotConsistentData, 'HistorySet Data accepts only a 1 Dimensional numpy array or a single value for method <_updateSpecializedInputValue>. Array shape is ' + str(value.shape)) #if value.size != 1 and not acceptArrayRealizations: self.raiseAnError(NotConsistentData,'HistorySet Data accepts only a numpy array of dim 1 or a single value for method <_updateSpecializedInputValue>. Size is ' + str(value.size)) unstructuredInput = True if value.size > 1 else False containerType = 'inputs' if not unstructuredInput else 'unstructuredInputs' if options and self._dataParameters['hierarchical']: # we retrieve the node in which the specialized 'History' has been stored parentID = None if type(name) == list: namep = name[1] if type(name[0]) == str: nodeId = name[0] else: if 'metadata' in options.keys(): nodeId = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: nodeId = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] else: if 'metadata' in options.keys(): nodeId = options['metadata']['prefix'] if 'parentID' in options['metadata'].keys(): parentID = options['metadata']['parentID'] else: nodeId = options['prefix'] if 'parentID' in options.keys(): parentID = options['parentID'] namep = name if parentID: tsnode = self.retrieveNodeInTreeMode(nodeId, parentID) else: tsnode = self.retrieveNodeInTreeMode(nodeId) self._dataContainer = tsnode.get('dataContainer') if not self._dataContainer: tsnode.add('dataContainer', { 'inputs': {}, 'unstructuredInputs': {}, 'outputs': {} }) self._dataContainer = tsnode.get('dataContainer') if namep in self._dataContainer[containerType].keys(): self._dataContainer[containerType].pop(name) if namep not in self._dataParameters['inParam']: self._dataParameters['inParam'].append(namep) self._dataContainer[containerType][namep] = c1darray( values=np.ravel(value)) # np.atleast_1d(np.array(value)) self.addNodeInTreeMode(tsnode, options) else: if type(name) == list: # there are info regarding the history number if name[0] in self._dataContainer[containerType].keys(): gethistory = self._dataContainer[containerType].pop( name[0]) gethistory[name[1]] = c1darray( values=np.ravel(np.array(value, dtype=float))) self._dataContainer[containerType][name[0]] = gethistory else: self._dataContainer[containerType][name[0]] = { name[1]: c1darray(values=np.ravel(np.array(value, dtype=float))) } else: # no info regarding the history number => use internal counter if len(self._dataContainer[containerType].keys()) == 0: self._dataContainer[containerType][1] = { name: c1darray(values=np.ravel(np.array(value, dtype=float))) } else: hisn = max(self._dataContainer[containerType].keys()) if name in list( self._dataContainer[containerType].values())[-1]: hisn += 1 self._dataContainer[containerType][hisn] = {} self._dataContainer[containerType][hisn][name] = c1darray( values=np.ravel(np.array( value, dtype=float))) # np.atleast_1d(np.array(value))
""" if abs(value - expected) > tol: print("checking answer", comment, value, "!=", expected) if updateResults: results["fail"] += 1 return False else: if updateResults: results["pass"] += 1 return True #establish test array origin = np.array([-3.14, 2.99792, 2.718, 8.987, 0.618]) #test init testArray = cached_ndarray.c1darray(values=origin) #test iter, getitem for i, val in enumerate(testArray): checkAnswer('content storage indexing', val, origin[i]) #test len checkAnswer('array length', len(testArray), 5) #test append single value testArray.append(-6.626) checkAnswer('append value', testArray[-1], -6.626) #test append array testArray.append(np.array([12.56, 6.67])) checkAnswer('append array, 0', testArray[-2], 12.56) checkAnswer('append array, 1', testArray[-1], 6.67)
def _specializedLoadXMLandCSV(self, filenameRoot, options): """ Function to load the xml additional file of the csv for data (it contains metadata, etc). It must be implemented by the specialized classes @ In, filenameRoot, string, file name root @ In, options, dict, dictionary -> options for loading @ Out, None """ #For HistorySet, create an XML file, and multiple CSV #files. The first CSV file has a header with the input names, #and a column for the filenames. There is one CSV file for each #data line in the first CSV and they are named with the #filename. They have the output names for a header, a column #for time, and the rest of the file is data for different times. if options is not None and 'fileToLoad' in options.keys(): name = os.path.join(options['fileToLoad'].getPath(), options['fileToLoad'].getBase()) else: name = self.name filenameLocal = os.path.join(filenameRoot, name) if os.path.isfile(filenameLocal + '.xml'): xmlData = self._loadXMLFile(filenameLocal) assert (xmlData["fileType"] == "HistorySet") if "metadata" in xmlData: self._dataContainer['metadata'] = xmlData["metadata"] mainCSV = os.path.join(filenameRoot, xmlData["filenameCSV"]) else: mainCSV = os.path.join(filenameRoot, name + '.csv') myFile = open(mainCSV, "rU") header = myFile.readline().rstrip() inpKeys = header.split(",")[:-1] inpValues = [] outKeys = [] outValues = [] allLines = myFile.readlines() for mainLine in allLines: mainLineList = mainLine.rstrip().split(",") inpValues_h = [utils.partialEval(a) for a in mainLineList[:-1]] inpValues.append(inpValues_h) dataFilename = mainLineList[-1] subCSVFilename = os.path.join(filenameRoot, dataFilename) myDataFile = open(subCSVFilename, "rU") subCSVFile = Files.returnInstance("CSV", self) subCSVFile.setFilename(subCSVFilename) self._toLoadFromList.append(subCSVFile) header = myDataFile.readline().rstrip() outKeys_h = header.split(",") outValues_h = [[] for a in range(len(outKeys_h))] for line in myDataFile.readlines(): lineList = line.rstrip().split(",") for i in range(len(outKeys_h)): outValues_h[i].append(utils.partialEval(lineList[i])) myDataFile.close() outKeys.append(outKeys_h) outValues.append(outValues_h) self._dataContainer['inputs'] = {} #XXX these are indexed by 1,2,... self._dataContainer['outputs'] = {} #XXX these are indexed by 1,2,... for i in range(len(inpValues)): mainKey = i + 1 subInput = {} subOutput = {} for key, value in zip(inpKeys, inpValues[i]): #subInput[key] = c1darray(values=np.array([value]*len(outValues[0][0]))) if key in self.getParaKeys('inputs'): subInput[key] = c1darray(values=np.array([value])) for key, value in zip(outKeys[i], outValues[i]): if key in self.getParaKeys('outputs'): subOutput[key] = c1darray(values=np.array(value)) self._dataContainer['inputs'][mainKey] = subInput self._dataContainer['outputs'][mainKey] = subOutput #extend the expected size of this point set self.numAdditionalLoadPoints = len(allLines) #used in checkConsistency self.checkConsistency()