def read(filename, vars, rqstTime): ''' Read a file or files from a directory given a wild-card expression ''' # @todo Reading a single file of netcdf cf convention now #cherrypy.log("vtkread " + filename + " " + vars + " " + str(time)) reader = vtk.vtkNetCDFCFReader() #get test data reader.SphericalCoordinatesOff() reader.SetOutputTypeToImage() reader.ReplaceFillValueWithNanOn() reader.SetFileName(filename) reader.UpdateInformation() #obtain temporal information rawTimes = reader.GetOutputInformation(0).Get(vtk.vtkStreamingDemandDrivenPipeline.TIME_STEPS()) tunits = reader.GetTimeUnits() converters = attrib_to_converters(tunits) # pick particular timestep if rqstTime is not None and rawTimes is not None: utcconverter = attrib_to_converters("days since 1970-0-0") abs_request_time = utcconverter[0](float(rqstTime)/(1000*60*60*24)) local_request_time = converters[5](abs_request_time) # For now clamp to time range if float(local_request_time) < rawTimes[0]: local_request_time = rawTimes[0] elif float(local_request_time) > rawTimes[-1]: local_request_time = rawTimes[-1] sddp = reader.GetExecutive() sddp.SetUpdateTimeStep(0, local_request_time) # enable only chosen array(s) narrays = reader.GetNumberOfVariableArrays() for x in range(0,narrays): arrayname = reader.GetVariableArrayName(x) if arrayname in vars: #cherrypy.log("Enable " + arrayname) reader.SetVariableArrayStatus(arrayname, 1) else: #cherrypy.log("Disable " + arrayname) reader.SetVariableArrayStatus(arrayname, 0) # wrap around to get the implicit cell extent = reader.GetOutputInformation(0).Get(vtk.vtkStreamingDemandDrivenPipeline.WHOLE_EXTENT()) pad = vtk.vtkImageWrapPad() reader.Update() data = reader.GetOutput() da = data.GetPointData().GetArray(0).GetName(); data.GetPointData().SetActiveScalars(da) pad.SetInputData(data) pad.SetOutputWholeExtent(extent[0], extent[1]+1, extent[2], extent[3], extent[4], extent[5]); # Convert to polydata sf = vtk.vtkDataSetSurfaceFilter() sf.SetInputConnection(pad.GetOutputPort()) # Error reading file? if not sf.GetOutput(): raise IOError("Unable to load data file: " + filename) # Convert to GeoJSON gw = vtk.vtkGeoJSONWriter() gw.SetInputConnection(sf.GetOutputPort()) gw.SetScalarFormat(2) gw.WriteToOutputStringOn() gw.Write() gj = str(gw.RegisterAndGetOutputString()).replace('\n','') return gj
def import_file(self, collection, filename, private=True): """Import metadata from a filename into the database This method reads a filename (fullpath) for its metadata and stores it into the specified collection of the database. :param collection: Name of the collection to look for basename. :type collection: str. :param filename: Name of the file with fullpath (eg. /home/clt.nc). :type filename: str. :param private: Should the entry be marked as private. :type private: bool """ if (not (os.path.isfile(filename) and os.path.exists(filename))): raise Exception("File " + filename + " does not exist") # @note Assuming that getting mongo collection everytime # is not going to cause much performance penalty coll = self._db[collection] print 'Begin importing %s into database' % filename variables = [] basename = os.path.basename(filename) filenamesplitted = os.path.splitext(basename) fileprefix = filenamesplitted[0] filesuffix = filenamesplitted[1] if self.is_exists(collection, filename): print 'Data %s already exists' % filename return if filesuffix == ".nc": # VTK is required import vtk reader = vtk.vtkNetCDFCFReader() reader.SphericalCoordinatesOff() reader.SetOutputTypeToImage() reader.ReplaceFillValueWithNanOn() reader.SetFileName(filename) reader.Update() data = reader.GetOutput() # Obtain spatial information bounds = data.GetBounds() # Obtain temporal information timeInfo = {} times = reader.GetOutputInformation(0).Get(vtk.vtkStreamingDemandDrivenPipeline.TIME_STEPS()) timeInfo['rawTimes'] = times #time steps in raw format tunits = reader.GetTimeUnits() timeInfo['units'] = tunits #calendar info needed to interpret/convert times converters = attrib_to_converters(tunits) if converters and times: timeInfo['numSteps'] = len(times) nativeStart = converters[3] timeInfo['nativeStart'] = nativeStart stepUnits = converters[2] timeInfo['nativeUnits'] = stepUnits stepSize = 0 if len(times) > 1: stepSize = times[1]-times[0] timeInfo['nativeDelta'] = stepSize stdTimeRange = (converters[0](times[0]), converters[0](times[-1])) timeInfo['nativeRange'] = (times[0], times[-1]) stdTimeDelta = 0 if len(times) > 1: stdTimeDelta = converters[0](times[1]) - converters[0](times[0]) timeInfo['stdDelta'] = stdTimeDelta stdTimeRange = (converters[0](times[0]), converters[0](times[-1])) timeInfo['stdTimeRange'] = stdTimeRange #first and last time as normalized integers dateRange = (converters[1](stdTimeRange[0]), converters[1](stdTimeRange[1])) timeInfo['dateRange'] = dateRange #first and last time in Y,M,D format # Obtain array information pds = data.GetPointData() pdscount = pds.GetNumberOfArrays() if times == None: times = [0] # Go through all timesteps to accumulate global min and max values for t in times: firstTStep = t==times[0] arrayindex = 0 # Go through all arrays for i in range(0, pdscount): pdarray = pds.GetArray(i) if not pdarray: # Got an abstract array continue if firstTStep: # Create new record for this array variable = {} else: # Extend existing record variable = variables[arrayindex] # Tell reader to read data so that we can get info about this time step sddp = reader.GetExecutive() sddp.SetUpdateTimeStep(0,t) sddp.Update() arrayindex = arrayindex + 1 if firstTStep: # Record unchanging meta information variable["name"] = pdarray.GetName() variable["dim"] = [] variable["tags"] = [] variable["units"] = reader.QueryArrayUnits(pdarray.GetName()) # Find min and max for each component of this array at this timestep componentCount = pdarray.GetNumberOfComponents() minmax = [] for j in range(0, componentCount): minmaxJ = [0,-1] pdarray.GetRange(minmaxJ, j) minmax.append(minmaxJ[0]) minmax.append(minmaxJ[1]) if firstTStep: # Remember what we learned about this new array variable["range"] = minmax variables.append(variable) else: # Extend range if necessary from this timesteps range for j in range(0, componentCount): if minmax[j*2+0] < variable["range"][j*2+0]: variable["range"][j*2+0] = minmax[j*2+0] if minmax[j*2+1] > variable["range"][j*2+1]: variable["range"][j*2+1] = minmax[j*2+1] # Record what we've learned in the data base insertId = coll.insert({"name":fileprefix, "basename":filename, "variables":variables, "timeInfo":timeInfo, "spatialInfo":bounds, "private":private}) print 'Done importing %s into database' % filename