def read(filename, vars, rqstTime):
''' Read a file or files from a directory given a wild-card expression
'''
# @todo Reading a single file of netcdf cf convention now
#cherrypy.log("vtkread " + filename + " " + vars + " " + str(time))
reader = vtk.vtkNetCDFCFReader() #get test data
reader.SphericalCoordinatesOff()
reader.SetOutputTypeToImage()
reader.ReplaceFillValueWithNanOn()
reader.SetFileName(filename)
reader.UpdateInformation()
#obtain temporal information
rawTimes = reader.GetOutputInformation(0).Get(vtk.vtkStreamingDemandDrivenPipeline.TIME_STEPS())
tunits = reader.GetTimeUnits()
converters = attrib_to_converters(tunits)
# pick particular timestep
if rqstTime is not None and rawTimes is not None:
utcconverter = attrib_to_converters("days since 1970-0-0")
abs_request_time = utcconverter[0](float(rqstTime)/(1000*60*60*24))
local_request_time = converters[5](abs_request_time)
# For now clamp to time range
if float(local_request_time) < rawTimes[0]:
local_request_time = rawTimes[0]
elif float(local_request_time) > rawTimes[-1]:
local_request_time = rawTimes[-1]
sddp = reader.GetExecutive()
sddp.SetUpdateTimeStep(0, local_request_time)
# enable only chosen array(s)
narrays = reader.GetNumberOfVariableArrays()
for x in range(0,narrays):
arrayname = reader.GetVariableArrayName(x)
if arrayname in vars:
#cherrypy.log("Enable " + arrayname)
reader.SetVariableArrayStatus(arrayname, 1)
else:
#cherrypy.log("Disable " + arrayname)
reader.SetVariableArrayStatus(arrayname, 0)
# wrap around to get the implicit cell
extent = reader.GetOutputInformation(0).Get(vtk.vtkStreamingDemandDrivenPipeline.WHOLE_EXTENT())
pad = vtk.vtkImageWrapPad()
reader.Update()
data = reader.GetOutput()
da = data.GetPointData().GetArray(0).GetName();
data.GetPointData().SetActiveScalars(da)
pad.SetInputData(data)
pad.SetOutputWholeExtent(extent[0], extent[1]+1,
extent[2], extent[3],
extent[4], extent[5]);
# Convert to polydata
sf = vtk.vtkDataSetSurfaceFilter()
sf.SetInputConnection(pad.GetOutputPort())
# Error reading file?
if not sf.GetOutput():
raise IOError("Unable to load data file: " + filename)
# Convert to GeoJSON
gw = vtk.vtkGeoJSONWriter()
gw.SetInputConnection(sf.GetOutputPort())
gw.SetScalarFormat(2)
gw.WriteToOutputStringOn()
gw.Write()
gj = str(gw.RegisterAndGetOutputString()).replace('\n','')
return gj
def import_file(self, collection, filename, private=True):
"""Import metadata from a filename into the database
This method reads a filename (fullpath) for its metadata
and stores it into the specified collection of the database.
:param collection: Name of the collection to look for basename.
:type collection: str.
:param filename: Name of the file with fullpath (eg. /home/clt.nc).
:type filename: str.
:param private: Should the entry be marked as private.
:type private: bool
"""
if (not (os.path.isfile(filename) and os.path.exists(filename))):
raise Exception("File " + filename + " does not exist")
# @note Assuming that getting mongo collection everytime
# is not going to cause much performance penalty
coll = self._db[collection]
print 'Begin importing %s into database' % filename
variables = []
basename = os.path.basename(filename)
filenamesplitted = os.path.splitext(basename)
fileprefix = filenamesplitted[0]
filesuffix = filenamesplitted[1]
if self.is_exists(collection, filename):
print 'Data %s already exists' % filename
return
if filesuffix == ".nc":
# VTK is required
import vtk
reader = vtk.vtkNetCDFCFReader()
reader.SphericalCoordinatesOff()
reader.SetOutputTypeToImage()
reader.ReplaceFillValueWithNanOn()
reader.SetFileName(filename)
reader.Update()
data = reader.GetOutput()
# Obtain spatial information
bounds = data.GetBounds()
# Obtain temporal information
timeInfo = {}
times = reader.GetOutputInformation(0).Get(vtk.vtkStreamingDemandDrivenPipeline.TIME_STEPS())
timeInfo['rawTimes'] = times #time steps in raw format
tunits = reader.GetTimeUnits()
timeInfo['units'] = tunits #calendar info needed to interpret/convert times
converters = attrib_to_converters(tunits)
if converters and times:
timeInfo['numSteps'] = len(times)
nativeStart = converters[3]
timeInfo['nativeStart'] = nativeStart
stepUnits = converters[2]
timeInfo['nativeUnits'] = stepUnits
stepSize = 0
if len(times) > 1:
stepSize = times[1]-times[0]
timeInfo['nativeDelta'] = stepSize
stdTimeRange = (converters[0](times[0]), converters[0](times[-1]))
timeInfo['nativeRange'] = (times[0], times[-1])
stdTimeDelta = 0
if len(times) > 1:
stdTimeDelta = converters[0](times[1]) - converters[0](times[0])
timeInfo['stdDelta'] = stdTimeDelta
stdTimeRange = (converters[0](times[0]), converters[0](times[-1]))
timeInfo['stdTimeRange'] = stdTimeRange #first and last time as normalized integers
dateRange = (converters[1](stdTimeRange[0]), converters[1](stdTimeRange[1]))
timeInfo['dateRange'] = dateRange #first and last time in Y,M,D format
# Obtain array information
pds = data.GetPointData()
pdscount = pds.GetNumberOfArrays()
if times == None:
times = [0]
# Go through all timesteps to accumulate global min and max values
for t in times:
firstTStep = t==times[0]
arrayindex = 0
# Go through all arrays
for i in range(0, pdscount):
pdarray = pds.GetArray(i)
if not pdarray:
# Got an abstract array
continue
if firstTStep:
# Create new record for this array
variable = {}
else:
# Extend existing record
variable = variables[arrayindex]
# Tell reader to read data so that we can get info about this time step
sddp = reader.GetExecutive()
sddp.SetUpdateTimeStep(0,t)
sddp.Update()
arrayindex = arrayindex + 1
if firstTStep:
# Record unchanging meta information
variable["name"] = pdarray.GetName()
variable["dim"] = []
variable["tags"] = []
variable["units"] = reader.QueryArrayUnits(pdarray.GetName())
# Find min and max for each component of this array at this timestep
componentCount = pdarray.GetNumberOfComponents()
minmax = []
for j in range(0, componentCount):
minmaxJ = [0,-1]
pdarray.GetRange(minmaxJ, j)
minmax.append(minmaxJ[0])
minmax.append(minmaxJ[1])
if firstTStep:
# Remember what we learned about this new array
variable["range"] = minmax
variables.append(variable)
else:
# Extend range if necessary from this timesteps range
for j in range(0, componentCount):
if minmax[j*2+0] < variable["range"][j*2+0]:
variable["range"][j*2+0] = minmax[j*2+0]
if minmax[j*2+1] > variable["range"][j*2+1]:
variable["range"][j*2+1] = minmax[j*2+1]
# Record what we've learned in the data base
insertId = coll.insert({"name":fileprefix, "basename":filename,
"variables":variables,
"timeInfo":timeInfo,
"spatialInfo":bounds,
"private":private})
print 'Done importing %s into database' % filename