def compVarSlice(fileA, fileB, var, dim, tol=0.0, start=0, end=0):
'''Compare a slice (of given dimension) through named variable'''
# open files
fpA = cdms.open(fileA)
fpB = cdms.open(fileB)
# check named variable present in both files
varsA = Set(fpA.listvariables())
varsB = Set(fpB.listvariables())
commonVars = varsA & varsB
if var not in commonVars:
fpA.close()
fpB.close()
return (FALSE, var + ' not common', varsA, varsB)
# ditto for named dimension
dimsA = Set(fpA.listdimension())
dimsB = Set(fpB.listdimension())
commonDims = dimsA & dimsB
if dim not in commonDims:
fpA.close()
fpB.close()
return (FALSE, dim + ' not common', dimsA, dimsB)
# get the slices
sliceA = eval(r"fpA('" + var + "'," + dim + "=slice(" + str(start) + "," +
str(end) + "))")
sliceB = eval(r"fpB('" + var + "'," + dim + "=slice(" + str(start) + "," +
str(end) + "))")
# close files
fpA.close()
fpB.close()
# ensure dimensions of slices correct
if sliceA.shape != sliceB.shape:
return (FALSE, 'different shapes', sliceA.shape, sliceB.shape)
if sliceA.shape[0] != end - start:
return (FALSE, 'slice size wrong', str(sliceA.shape[0]),
str(end - start))
if sliceA.shape[0] == 0:
return (FALSE, 'slice size zero', str(sliceA.shape[0]),
str(end - start))
# make actual comparison
maxDelta = MV.maximum(abs(sliceA - sliceB))
if maxDelta > tol:
return (FALSE, 'max diff > ' + str(tol), '', '')
else:
return (TRUE, '', '', '')
def _test(filename, varname=None):
try:
import cdms
except ImportError:
import cdms2 as cdms
f = cdms.open(filename)
if varname:
var = f[varname]
else:
var = f.getVariables()[0]
axes = var.getAxisList()
rlons = axes[3]
rlats = axes[2]
lonBounds = rlons.getBounds()
latBounds = rlats.getBounds()
print "file longitudes"
print lonBounds
print "finding longitudes"
for lon in range(360):
print lon, findIndexLon(lon, lonBounds, useLimits=True)
print "file latitudes"
print latBounds
print "finding latitudes"
for lat in range(-90, 91):
print lat, findIndex(lat, latBounds, useLimits=True)
def doRegrid(infile, varname, nodata, outfile, outformat, options, xRange, yRange, gt):
# read infile
a=cdms.open(infile)
start_time=a.getAxis('time').asComponentTime()[]
end_time=a.getAxis('time').asComponenentTime()[-1]
data=a()
if fh is None:
exitMessage("Could not open file {0}. Exit 2.".format(infile), 2)
print fh.history
# read variable
if varname not in fh.variables.keys():
exitMessage('variable named '+varname+' could not be found. Exit 4.', 4)
yVar=numpy.array(fh.variables[varname][:])
# read lat
lat=numpy.array(fh.variables['lat'][:])
# read lon
lon=numpy.array(fh.variables['lon'][:])
# if arrays are 2-d, 1 lat/lon per pixel
if len(lat.shape)==2:
if (lat.shape[0]==yVar.shape[1]) and (lat.shape[1]==yVar.shape[2]):
points = numpy.array( [numpy.ravel(lon), numpy.ravel(lat)] ).T
elif (lat.shape[0]==yVar.shape[1]) and (lat.shape[1]==2):
avgLon = numpy.repeat( numpy.array( [ numpy.ravel( 0.5*(lon[:,0]+lon[:,1]) ) ] ), lat.shape[0], 0)
avgLat = numpy.repeat( numpy.array( [ numpy.ravel( 0.5*(lat[:,0]+lat[:,1]) ) ] ).T, lon.shape[0], 1)
points = numpy.array( [numpy.ravel(avgLon), numpy.ravel(avgLat)] ).T
else:
exitMessage("Unknown case.", 2)
elif len(lat.shape)==1:
newLon = numpy.repeat( [ lon ] , lat.shape[0], 0)
newLat = numpy.repeat( [ lat.T ] , lon.shape[0], 1)
points = numpy.array( [numpy.ravel(newLon), numpy.ravel(newLat)] ).T
else:
exitMessage('Unknown structure for lat/lon. Exit(3).', 3)
xi=[]
for ix in xRange:
for iy in yRange:
xi.append([ix, iy])
# instantiate a file
outDrv = gdal.GetDriverByName(outformat)
outDS = outDrv.Create(outfile, len(xRange), len(yRange), yVar.shape[0], GDT_Float32, options)
outDS.SetProjection(latlon())
outDS.SetGeoTransform(gt)
# interpolate
for iband in range(yVar.shape[0]):
yInt = interpolate.griddata(points, yVar[iband, :, :].ravel(), xi, method='linear', fill_value=nodata)
yIntToWrite = yInt.reshape(len(xRange), len(yRange)).copy()
# save to file
outDS.GetRasterBand( iband + 1 ).WriteArray( numpy.flipud(yIntToWrite.T), 0, 0)
gdal.TermProgress_nocb( (iband+1)/float(yVar.shape[0]) )
gdal.TermProgress_nocb(1)
def compute(self) :
args = inspect.getargspec(cdms.open)
def_args = args[3]
uri = None
mode = def_args[0]
template = def_args[1]
dods = def_args[2]
if not self.hasInputFromPort('uri') :
print "Error: must have uri input"
return
if self.hasInputFromPort('uri') :
inuri = self.getInputFromPort('uri')
uri = os.path.join(sys.prefix, inuri)
if self.hasInputFromPort('mode') :
mode = self.getInputFromPort('mode')
if self.hasInputFromPort('template') :
template = self.getInputFromPort('template')
if self.hasInputFromPort('dods') :
dods = self.getInputFromPort('dods')
# output the cdmsfile object.
cdmsfile = cdms.open(uri,mode,template,dods)
output = cdms_dataset(cdmsfile)
self.setResult("cdms_dataset", output)
def compute(self):
args = inspect.getargspec(cdms.open)
def_args = args[3]
uri = None
mode = def_args[0]
template = def_args[1]
dods = def_args[2]
if not self.has_input('uri'):
print "Error: must have uri input"
return
if self.has_input('uri'):
inuri = self.get_input('uri')
uri = os.path.join(sys.prefix, inuri)
if self.has_input('mode'):
mode = self.get_input('mode')
if self.has_input('template'):
template = self.get_input('template')
if self.has_input('dods'):
dods = self.get_input('dods')
# output the cdmsfile object.
cdmsfile = cdms.open(uri, mode, template, dods)
output = cdms_dataset(cdmsfile)
self.set_output("cdms_dataset", output)
def compVarSlice(fileA, fileB, var, dim, tol=0.0, start=0, end=0):
'''Compare a slice (of given dimension) through named variable'''
# open files
fpA = cdms.open(fileA)
fpB = cdms.open(fileB)
# check named variable present in both files
varsA = Set(fpA.listvariables())
varsB = Set(fpB.listvariables())
commonVars = varsA & varsB
if var not in commonVars:
fpA.close()
fpB.close()
return (FALSE,var+' not common',varsA,varsB)
# ditto for named dimension
dimsA = Set(fpA.listdimension())
dimsB = Set(fpB.listdimension())
commonDims = dimsA & dimsB
if dim not in commonDims:
fpA.close()
fpB.close()
return (FALSE,dim+' not common',dimsA,dimsB)
# get the slices
sliceA = eval(r"fpA('"+var+"',"+dim+"=slice("+str(start)+","+str(end)+"))")
sliceB = eval(r"fpB('"+var+"',"+dim+"=slice("+str(start)+","+str(end)+"))")
# close files
fpA.close()
fpB.close()
# ensure dimensions of slices correct
if sliceA.shape != sliceB.shape:
return (FALSE,'different shapes',sliceA.shape,sliceB.shape)
if sliceA.shape[0] != end - start:
return (FALSE,'slice size wrong',str(sliceA.shape[0]),str(end-start))
if sliceA.shape[0] == 0:
return (FALSE,'slice size zero',str(sliceA.shape[0]),str(end-start))
# make actual comparison
maxDelta = MV.maximum(abs(sliceA - sliceB))
if maxDelta > tol:
return (FALSE,'max diff > '+str(tol),'','')
else:
return (TRUE,'','','')
def get_html(self):
if self.ddd.filetype == 'nc':
out = 'nc\n'
try:
import cdms
self.cdmsv = 1
except:
import cdms2 as cdms
self.cdmsv = 2
try:
nc = cdms.open( self.ddd.this_file, 'r' )
gl = nc.listglobal()
gl.sort()
stem = string.split( string.split( self.ddd.this_file, '/' )[-1], '.')[0]
bits = string.split( stem, '_' )
out = '<h2>Global attributes</h2>\n<ul>\n'
for g in gl:
gv = nc.getglobal( g )
out += '<li>%s: %s</li>\n' % (g,str(gv))
out += '</ul><br/>\n\n'
scenario = nc.getglobal( 'scenario_tag' )
model = nc.getglobal( 'model_tag' )
startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx, bounds='clim_bounds' )
trange = '%s-%s' % (startYear, endYear)
self.time_info = (startYear, startMonth, startDay, endYear)
vk = nc.variables.keys()
vk.sort()
for v in vk:
Axes = map( lambda x: x.attributes.get('standard_name','-'), nc.variables[v].getAxisList() )
axesstring = string.join( Axes, ', ' )
out += '<h2>%s [ %s ]</h2>' % (v,axesstring)
out += '<a href="/cgi-bin/ncbv/%s/%s">View data</a>\n' % (self.path,v)
ak = nc.variables[v].attributes.keys()
ak.sort()
out += '<ul>\n'
for a in ak:
out += '<li>\n%s::%s\n</li>' % (a,nc.variables[v].attributes[a])
if len( nc.variables[v].shape ) == 3:
standard_name = nc.variables[v].attributes.get('standard_name', nc.variables[v].attributes['name'] )
out += '</ul>\n'
out += '<br/>%s, %s, %s, %s\n' % (standard_name, model, scenario, trange)
nc.close()
except:
out += 'failed to construct html'
raise
return out
def scanFAAM(fileName=None,
vars=None,
nth=4,
missingValuesToUse=(-9999., -32767.)):
"""
Scans every 'nth' variable in the list of variables (or found in the
file and gets the first and last index of the first (time) dimension
that holds real (non-missing) values.
"""
if type(missingValuesToUse) != type((1, 2)):
missingValuesToUse = (missingValuesToUse, )
startList = []
endList = []
start = None
end = None
if not fileName and not vars:
raise "You must provide either a file name or a list of cdms variables."
if fileName:
f = cdms.open(fileName)
vars = f.listvariables()
for var in vars:
if type(var) != type(""):
id = var.id
else:
id = var
if id[-4:] == "FLAG" or id == "Time":
continue
if type(var) == type(""):
var = f(var)
step = 1000
while (start, end) == (None, None):
(start, end) = findMissing(var, step, missingValuesToUse)
step = step / 2
startList.append(start)
endList.append(end)
print "Start/End index: %s %s:%s" % (id, start, end)
startMin = min(startList)
endMax = max(endList)
return (startMin, endMax)
def selectFeature(self, featureId, bbox, dimensionSpec):
cacheKey = (featureId, self._makeDimKey(dimensionSpec))
try:
ncfilename = self._map[cacheKey]
except KeyError:
(fd, ncfilename) = tempfile.mkstemp('.nc', featureId+'_', self._tempdir)
os.close(fd)
feature = self.dataset.getFeature(featureId)
self._extract(feature, dimensionSpec, ncfilename)
#self._map[cacheKey] = ncfilename
d = cdms.open(ncfilename)
(lon1, lat1, lon2, lat2) = bbox
var = d(featureId, latitude=(lat1, lat2), longitude=(lon1, lon2), squeeze=1)
d.close()
return var
def check_open(self):
if self.openned:
return True
try:
file = self.ddd.this_file
html = '%s <br/>' % file
try:
import cdms
self.nc = cdms.open( file, 'r' )
self.cdmsv = 1
except:
import cdms2
self.nc = cdms2.open( file, 'r' )
self.cdmsv = 2
self.openned = True
return True
except:
return False
def scanFAAM(fileName=None, vars=None, nth=4, missingValuesToUse=(-9999., -32767.)):
"""
Scans every 'nth' variable in the list of variables (or found in the
file and gets the first and last index of the first (time) dimension
that holds real (non-missing) values.
"""
if type(missingValuesToUse)!=type((1,2)):
missingValuesToUse=(missingValuesToUse,)
startList=[]
endList=[]
start=None
end=None
if not fileName and not vars:
raise "You must provide either a file name or a list of cdms variables."
if fileName:
f=cdms.open(fileName)
vars=f.listvariables()
for var in vars:
if type(var)!=type(""):
id=var.id
else:
id=var
if id[-4:]=="FLAG" or id=="Time":
continue
if type(var)==type(""):
var=f(var)
step=1000
while (start, end)==(None, None):
(start, end)=findMissing(var, step, missingValuesToUse)
step=step/2
startList.append(start)
endList.append(end)
print "Start/End index: %s %s:%s" % (id, start, end)
startMin=min(startList)
endMax=max(endList)
return (startMin, endMax)
def getVariable(infile, var, lat, lon):
"""Get the variable needed for the location (lat, lon)."""
datafile=cdms.open(infile)
if var==None:
var=datafile.listvariables()[0]
metadata=datafile[var]
latax=metadata.getLatitude()[:]
lonax=metadata.getLongitude()[:]
lat=nudgeSingleValuesToAxisValues(lat, latax, "latitude")[0]
lon=nudgeSingleValuesToAxisValues(lon, lonax, "longitude")[0]
data=datafile(var, lat=lat, lon=lon, squeeze=1)
print data.id
data=data(time=slice(0,tlen))
data=accumulate24Hourly(data)
datafile.close()
# Do some working with the data depending on the variable
if var=="tmpk": # ncep temperature
data[:]=data[:]-270.
data.units="degC"
elif var=="apcp": # ncep rainfall
# fix < zeros
newarray=[]
count=0
for i in data:
newarray.append([])
for x in i:
x=x[0]
if x<0.0:
x=0.0
newarray[count].append(x)
count=count+1
data[:]=Numeric.array(newarray, 'f')
return (data,lat,lon)
def processRainfall(file, outdir, var, north, west, south, east):
"Subsets, averages, writes to binary files."
f=cdms.open(file)
v=f(var, lat=(south, north), lon=(west, east))
timevalues=v.getTime()[:]
t0=timevalues[0]
# I need to test if step 0 always has only missing values
# remove -50 values???
v=MA.masked_less(v,0)
# create average of all ensemble members
av=MA.average(v, axis=1)
# get stuff for name
datetime=os.path.split(file)[-1].split(".")[1]
outpaths=[]
# now step through time dimension (0)
count=0
for dslice in av:
ts=timevalues[count]-t0
outfile="rainfall.%s.%dh.dat" % (datetime, ts)
outpath=os.path.join(outdir, outfile)
count=count+1
numarray=Numeric.array(dslice._data)
sh=numarray.shape
length=sh[0]*sh[1]
flatarray=Numeric.resize(numarray, [length])
output=open(outpath, "wb")
arr=array.array('f', flatarray)
arr.tofile(output)
output.close()
print "Written:", outpath
outpaths.append(outpath)
return outpaths
import sys
import cdms
import spanlib
import MV
# sys.path.insert(0,'../src/build/lib.linux-i686-2.4')
cdms.axis.latitude_aliases.append("Y")
cdms.axis.longitude_aliases.append("X")
cdms.axis.time_aliases.append("T")
f = cdms.open("../example/data2.cdf")
s2 = f("ssta", latitude=(-10, 10), longitude=(110, 180))
s1 = f("ssta", latitude=(-15, 15), longitude=(210, 250))
print "Data in:", s1.shape, s2.shape
res = spanlib.stackData(s1, s2)
print res[0].shape
SP = spanlib.SpAn(MV.array(res[0]), weights=MV.array(res[1]))
eof, pc, ev = SP.pca()
## for ax in res[3]:
## ax[0]=eof.getAxis(0)
import cdms, time, MA
f = cdms.open('test0.nc', 'a')
t = f.variables['air_temperature']
print time.time()
x = t.getValue()
for i in range(t.shape[0]):
for j in range(t.shape[1]):
x[i, j, :] += 2.
t[:, :, :] = x
print time.time()
for i in range(t.shape[0]):
for j in range(t.shape[1]):
t[i, j, :] = MA.array(t[i, j, :] + 2., 'f')
print time.time()
f.close()
# Import the modules needed for the tutorial
import vcs, cdms, cdutil, time, os, sys
# Open data file:
filepath = os.path.join(sys.prefix, 'sample_data/clt.nc')
cdmsfile = cdms.open( filepath )
# Extract a 3 dimensional data set and get a subset of the time dimension
data = cdmsfile('clt', longitude=(-180, 180), latitude = (-90., 90.))
# Initial VCS.
v = vcs.init()
# Show the list of persistent isofill graphics methods.
v.show('isofill')
# Assign the variable "cf_asd" to the persistent 'ASD' isofill graphics methods.
cf_asd = v.getisofill( 'ASD' )
# Plot the data using the above boxfill graphics method.
v.plot( data, cf_asd )
print ""
print "Press the Return key to see next plot."
sys.stdin.readline()
# List the 'ASD' isofill graphics methods attributes.
cf_asd.list()
# change the isofill levels and the color indices.
cf_asd.levels = ( [0,20],[20,40],[50,60],[60,70],[70,80],[90,100])
cf_asd.fillareacolors=( [22,44,66,88,110,132])
# import needed modules
import cdms, vcs, cdutil, genutil, cdtime, MA, MV, sys, os
# get an xml file or just a netcdf file
file = os.path.join(sys.prefix,'sample_data/tas_ccsr-95a.xml')
a=cdms.open(file)
# get the data and print out it's shape
data=a('tas')
print data.shape
#(12, 1, 32, 64)
#
# get the list of global attributes and put them into a dictionary
list_file=a.attributes.keys()
file_dic={}
for i in range(0,len(list_file)):
file_dic[i]=list_file[i],a.attributes[list_file[i] ]
# see what is in a list and a dictionary
print list_file
print file_dic
# now get the variable 'data' attributes and put into another dictionary
list_data=data.attributes.keys()
data_dic={}
for i in range(0,len(list_data)):
data_dic[i]=list_data[i],data.attributes[list_data[i] ]
# print the list and the dictionary
print list_data
print data_dic
######################### # Simple netcdf plotter # ######################### # Needed modules import vcs, sys, cdms # Arguments if len(sys.argv) < 3: print 'Usage: python quickview.py <filename> <varname>' sys.exit(1) filename = sys.argv[1] varname = sys.argv[2] # Open netcdf file f=cdms.open(filename) # Read our variable s=f(varname) # Create vcs canvas x=vcs.init() # Plot it x.plot(s)
def nc_lon_cyc(ifile, ofile, verbose=0):
ii = os.popen('cp %s %s' % (ifile, ofile), 'r')
ii.readlines()
ii.close()
ii = os.popen('chmod 777 %s' % ofile, 'r')
ii.readlines()
ii.close()
f = cdms.open(ofile, 'a')
oldLonAxis = f.getAxis('longitude')
if oldLonAxis.attributes.has_key('bounds'):
oldLonBounds = oldLonAxis.attributes['bounds']
axisMappings, nShift, lonValShift = getNewLonAxis(oldLonAxis)
nlon = len(oldLonAxis)
if nlon != nShift * 2:
print 'can only cope with shifts of half total length'
print nlon, nShift
raise 'config error'
for k in range(len(oldLonAxis)):
f.getAxis('longitude')[k] = axisMappings.getValue()[k]
##
## loop through variables and cycle on longitude axis.
## NB: only a limited number of variable shapes are supported.
##
for var in f.getVariables():
oldLonAxis = var.getLongitude()
if oldLonAxis:
newAxes = var.getAxisList()
if var.name == oldLonBounds:
##
## the cf-checker likes the bounds to include the coordinate value in
## an absolute sense, so that bounds for -180. should be, e.g., [-182, -178],
## NOT [178,-178].
##
if verbose > 0:
print 'shifting bounds'
print dir(var), var.getShape()
##
## shift longitude bounds by same amount as longitudes.
##
va = var.getValue()
for j in range(var.getShape()[0]):
va[j, :] += lonValShift[j]
var[:, :] = va
if verbose > 1:
print var.name, var
index = newAxes.index(oldLonAxis)
rank = var.rank()
aa = var.getValue()
if rank == 3 and index == 2:
for i in range(nShift):
var[:, :, i + nShift] = aa[:, :, i]
var[:, :, i] = aa[:, :, i + nShift]
elif rank == 2 and index == 0:
for i in range(nShift):
var[i + nShift, :] = aa[i, :]
var[i, :] = aa[i + nShift, :]
else:
print rank, index
raise 'rank index combination not programmed for'
if verbose > 0:
print var.name, index, rank
f.history += '; longitude cycled to start at 180 degrees west'
f.close()
#!/usr/bin/env python
import cdms, vcs, cdutil, genutil, os, sys, Numeric
from cdms import MV
f_in1 = cdms.open('./pot_temp_geopotential.nc')
f_in2 = cdms.open('./press_rho_temp.nc')
print
print "Reading in variables"
theta = f_in1('theta')
geopotential = f_in1('geopotential')
pl = f_in2('pl')
print
print "Truncating variables"
theta = cdms.createVariable(theta[:, :, 0, :], copy=1)
geopotential = cdms.createVariable(geopotential[:, :, 0, :], copy=1)
pl = cdms.createVariable(pl[:, :, 0, :], copy=1)
print
print "Calculating vpgf"
exec(open('calc_vpgf.txt'))
f_out = cdms.open('./vpgf.nc', 'w')
print
print "Writing vpgf to file"
f_out.write(vpgf)
print
print "Done"
# Import modules
import cdms, cdutil, MA, vcs, cdtime
import string, Numeric, time, MV, sys, os
from regrid import Regridder
from genutil import statistics
file1 = os.path.join(sys.prefix, 'sample_data/era40_tas_sample.nc')
f1 = cdms.open( file1 )
f1.showvariable()
f1.listdimension()
print f1.getAxis('time').asComponentTime()[0]
# 1990-1-1 0:0:0.0
print f1.getAxis('time').asComponentTime()[-1]
# 1993-12-1 0:0:0.0
file2 = os.path.join(sys.prefix, 'sample_data/era15_tas_sample.nc')
f2 = cdms.open( file2 )
f2.showvariable()
f2.listdimension()
print f2.getAxis('time').asComponentTime()[0]
# 1989-1-1 0:0:0.0
print f2.getAxis('time').asComponentTime()[-1]
# 1994-2-1 0:0:0.0
# get data with overlapping in overlapping time range
data1 = f1('tas', time = ('1991-1-1','1993-12-1'))
import cdms
from MA import allclose
f = cdms.open('clt.nc')
v = f('clt')
s = cdms.selectors.timeslice(0,120,12)
x = s.select(v)
print x.shape
y = v(s)
print y.shape
assert allclose(x,y)
assert allclose(x, f('clt', time=slice(0,120,12)))
x = v(s, longitude=(100,150))
print x.shape
y = x(latitude=(-40,40))
print y.shape
w = f('clt', cdms.selectors.longitude(100,140))
print w.shape
print f('clt', slice(0,1), squeeze=1).shape
#
# variable we want to work with
var='tas'
#
# models we will be using
model=['ccsr-95a', 'dnm-95a' ]
#
# set up a description string for addition to the global
# attributes in the output netcdf
#
model_description=''
# loop over all models--similar to a fortran do loop
for i in range(0,len(model)):
file_xml = os.path.join(sys.prefix,'sample_data/'+var+'_'+model[i]+'.xml') #a=cdms.open('/pcmdi/AMIP3/amip/mo/'+var+'/'+model[i]+'/'+var+'_'+model[i]+'.xml')
a=cdms.open(file_xml)
data=a[var]
print '---- ', i, model[i],data.shape
start_time = data.getTime().asComponentTime()[0]
end_time = data.getTime().asComponentTime()[-1]
print 'start time: ',start_time,' end time:',end_time
time_len = len(data.getTime())
print 'time axis lenght: ', time_len
a.close()
dm=str(i)+' = '+model[i]
model_description=model_description+', '+dm
#
print;print '__________________';print
# set up an output array for the global time series
glan=MA.zeros([len(model),time_len],MA.Float)
def simpleComp(fileA, fileB, tol=0.0):
'''A simple comparison function.
Attribute names and values are compared first.
Then the data arrays are compared within a
tolerance on a cell by cell basis.'''
# open files
fpA = cdms.open(fileA)
fpB = cdms.open(fileB)
# == global attributes ==
# compare attribute names
message = 'global attributes: '
globalNamesA = Set(fpA.listglobal())
globalNamesB = Set(fpB.listglobal())
symmetricDiff = globalNamesA ^ globalNamesB
if len(symmetricDiff) != 0:
(detailA, detailB) = setDiff(globalNamesA, globalNamesB, symmetricDiff)
return (FALSE, message, detailA, detailB)
# compare values
for globalName in globalNamesA:
# limit our checks to attributes with string values
if isinstance(eval(r'fpA.' + globalName), types.StringType):
globalValueA = eval(r'fpA.' + globalName)
globalValueB = eval(r'fpB.' + globalName)
if globalValueA != globalValueB:
message += globalName + ' values'
return (FALSE, message, globalValueA, globalValueB)
# == dimensions ==
# compare dimension names
dimNamesA = Set(fpA.listdimension())
dimNamesB = Set(fpB.listdimension())
symmetricDiff = dimNamesA ^ dimNamesB
if len(symmetricDiff) != 0:
message = 'dimensions:'
(detailA, detailB) = setDiff(dimNamesA, dimNamesB, symmetricDiff)
return (FALSE, message, detailA, detailB)
# loop over dimensions
for dimName in dimNamesA:
message = 'dimensions: ' + dimName
# compare attribute names
dimAttNamesA = Set(fpA[dimName].attributes.keys())
dimAttNamesB = Set(fpA[dimName].attributes.keys())
symmetricDiff = dimAttNamesA ^ dimAttNamesB
if len(symmetricDiff) != 0:
(detailA, detailB) = setDiff(dimAttNamesA, dimAttNamesB,
symmetricDiff)
return (FALSE, message, detailA, detailB)
# compare attribute values
for dimAttName in dimAttNamesA:
# assuming objects we can compare
dimAttValueA = eval(r"fpA['" + dimName + r"']." + dimAttName)
dimAttValueB = eval(r"fpB['" + dimName + r"']." + dimAttName)
if dimAttValueA != dimAttValueB:
message += ': ' + dimAttName
return (FALSE, message, dimAttValueA, dimAttValueB)
# compare data
dimDataShapeA = MV.shape(fpA[dimName])
dimDataShapeB = MV.shape(fpB[dimName])
if dimDataShapeA != dimDataShapeB:
message += ': data array shape'
return (FALSE, message, str(dimDataShapeA), str(dimDataShapeB))
maxDelta = MV.maximum(abs(fpA[dimName][:] - fpB[dimName][:]))
if maxDelta > tol:
message += ': delta: ' + str(maxDelta) + ' > ' + str(tol)
return (FALSE, message, '', '')
# == variables ==
# compare variable names
varNamesA = Set(fpA.listvariables())
varNamesB = Set(fpB.listvariables())
symmetricDiff = varNamesA ^ varNamesB
if len(symmetricDiff) != 0:
message = 'variables:'
(detailA, detailB) = setDiff(varNamesA, varNamesB, symmetricDiff)
return (FALSE, message, detailA, detailB)
# loop over variables
for varName in varNamesA:
message = 'variables: ' + varName
# compare attribute names
varAttNamesA = Set(fpA[varName].attributes.keys())
varAttNamesB = Set(fpA[varName].attributes.keys())
symmetricDiff = varAttNamesA ^ varAttNamesB
if len(symmetricDiff) != 0:
(detailA, detailB) = setDiff(varAttNamesA, varAttNamesB,
symmetricDiff)
return (FALSE, message, detailA, detailB)
# compare attribute values
for varAttName in varAttNamesA:
# assuming objects we can compare
varAttValueA = eval(r"fpA['" + varName + r"']." + varAttName)
varAttValueB = eval(r"fpB['" + varName + r"']." + varAttName)
if varAttValueA != varAttValueB:
message += ': ' + varAttName
return (FALSE, message, varAttValueA, varAttValueB)
# compare data
varDataShapeA = MV.shape(fpA[varName])
varDataShapeB = MV.shape(fpB[varName])
if varDataShapeA != varDataShapeB:
message += ': data array shape'
return (FALSE, message, str(varDataShapeA), str(varDataShapeB))
maxDelta = MV.maximum(abs(fpA[varName][:] - fpB[varName][:]))
if maxDelta > tol:
message += ': delta: ' + str(maxDelta) + ' > ' + str(tol)
return (FALSE, message, '', '')
# close files
fpA.close()
fpB.close()
return (TRUE, '', '', '')
def extract_ij(ifile, ofile, i, j, verbose=0):
oo = open(ofile, 'w')
f = cdms.open(ifile, 'r')
LonAxis = f.getAxis('longitude')
LatAxis = f.getAxis('latitude')
timeAxis = f.getAxis('time')
time = f.getAxis('time').getValue()
nlon = len(LonAxis)
nlat = len(LatAxis)
if i > nlon - 1:
raise 'request is beyong end of longitude range'
if j > nlat - 1:
raise 'request is beyong end of latitude range'
oo.write('"Time units","%s",,,,\n' % timeAxis.attributes['units'])
oo.write('"Longitude",%s,%s,\n' %
(LonAxis.getValue()[i], LonAxis.attributes['units']))
oo.write('"Latitude",%s,%s,\n' %
(LatAxis.getValue()[j], LatAxis.attributes['units']))
if LonAxis.attributes.has_key('bounds'):
LonBounds = LonAxis.attributes['bounds']
t = f.variables[LonBounds].getValue()
oo.write('"Longitudinal bounds","%s",%s,%s,\n' %
(LonBounds, t[i, 0], t[i, 1]))
if LatAxis.attributes.has_key('bounds'):
LatBounds = LatAxis.attributes['bounds']
t = f.variables[LatBounds].getValue()
oo.write('"Latitudinal bounds","%s",%s,%s,\n' %
(LatBounds, t[j, 0], t[j, 1]))
##
## loop through variables and cycle on longitude axis.
## NB: only a limited number of variable shapes are supported.
##
for var in f.getVariables():
Lon = var.getLongitude()
Lat = var.getLatitude()
if Lat and Lon:
Axes = var.getAxisList()
ii = Axes.index(LonAxis)
jj = Axes.index(LatAxis)
rank = var.rank()
aa = var.getValue()
print var.shape
oo.write('Name, "%s",\n' % (var.attributes['name']))
oo.write('Units, "%s",\n' % (var.attributes['units']))
oo.write('Standard name, "%s",\n' %
(var.attributes['standard_name']))
if rank == 3 and ii == 1 and jj == 2:
for k in range(len(time)):
oo.write('%s, %s,\n' % (time[k], aa[k, i, j]))
if rank == 3 and ii == 2 and jj == 1:
for k in range(len(time)):
print time[k], k, j, i
print aa[k, j, i]
oo.write('%s, %s, %s,\n' % (time[k], mn[k], aa[k, j, i]))
else:
print rank, ii, jj
print 'rank index combination not programmed for'
if verbose > 0:
print var.name, index, rank
f.close()
import MSU,cdms
weights_file='../Data/weights.nc'
f=cdms.open(weights_file)
w=f('weights') # There's 3 channel on these weights => MSU computed for each one at once
f.close()
critw=50. # Criteria to set to missing if too many missing values
file='../Data/ta.nc'
fout='equivalent_msu.nc'
f=cdms.open(file)
ta=f('ta')
tc=ta.getTime().asComponentTime()
print 'Input data:',ta.shape
print 'Time span:',tc[0],tc[-1]
print 'Computing Equivalent MSU temperatures'
msu=MSU.msu(ta,w,critw)
print 'MSU computed, it has 3 channels as did the weights input'
msu.info()
fout=cdms.open(fout,'w')
fout.write(msu[...,0],typecode='f',id='tam2')
fout.write(msu[...,1],typecode='f',id='tam4')
fout.write(msu[...,2],typecode='f',id='tam6')
fout.close()
print 'Finished'
def cdms2na(ncfile, na_file_names, naVars={}, variables=None, nFilesOnly="no",
rule=None, ffi="automatic", delimiter=" ", float_format="%g",
rules=None, sizeLimit=None):
"""
Main conversion function that calls the appropriate classes and functions
to write a NASA Ames file.
"""
#print infilename, outfilenames, nFilesOnly, naVars, variables
if type(na_file_names) == type("string"):
na_file_names = [na_file_names]
# Get which NASA Ames internal variables are allowed to be overwritten in the output files (i.e. by user inputs)
allowedOverwriteMetadata = ("DATE", "RDATE", "ANAME", "MNAME",
"ONAME", "ORG", "SNAME", "VNAME")
arrayArgs=["DATE", "RDATE", "ANAME", "VNAME"]
# ANAME[a] - array of 'a' x ANAME strings - aux var names
# DATE (array of three) - UT date at which the data within the file starts
# MNAME - mission name
# ONAME - name of originator(s)
# ORG - org or affiliation of originator(s)
# RDATE (array of three) - date of data reduction or revision
# SNAME - source of measurement or model output VNAME[n] - array of 'n' x
# VNAME strings - var names.
outputMessage=[]
msg="Reading data from: %s\n" % infilename
print msg
outputMessage.append(msg)
cdmsfile=cdms.open(infilename)
globals=cdmsfile.attributes
vars=[]
if not variables:
variables=cdmsfile.listvariables()
#for var in cdmsfile.listvariables():
#vars.append(cdmsfile(var))
for variable in variables:
varObj=cdmsfile(variable)
# Deal with singleton variables
if not hasattr(varObj, "rank"):
varMetadata=cdmsfile[variable].attributes
varValue=varObj
#print varMetadata, varValue, varMetadata.keys(), varMetadata._obj_.id
varObj=cdms.createVariable(Numeric.array(varObj), id=getBestName(varMetadata).replace(" ", "_"), attributes=varMetadata)
#print varObj, dir(varObj); sys.exit()
varObj.value=varObj._data[0]
#varObj.rank=0
#print varObj, varObj.attributes
vars.append(varObj)
# Re-order variables if they have the attribute 'nasa_ames_var_number'
orderedVars=[None]*1000
otherVars=[]
for var in vars:
varMetadata=cdmsfile[var]
if hasattr(varMetadata, "nasa_ames_var_number"):
num=varMetadata.nasa_ames_var_number
orderedVars[num]=var
else:
otherVars.append(var)
vars=[]
for var in orderedVars:
if var!=None:
vars.append(var)
vars=vars+otherVars
builder=NAContentCollector(vars, globals, rule=rule, cdmsfile=cdmsfile)
#print builder.na_dict["X"]
builtNADicts=[[builder.na_dict, builder.varIDs]]
if builder.varIDs==None:
msg="\nNo files created after variables parsed."
print msg
outputMessage.append(msg)
return outputMessage
while len(builder.varBin)>0:
builder=NAContentCollector(builder.varBin, globals, rule=rule, cdmsfile=cdmsfile)
outputMessage=outputMessage+builder.outputMessage
if builder.varIDs!=None: builtNADicts.append([builder.na_dict, builder.varIDs])
# Return only filenames if only want to know them now.
ncount=1
fileNames=[]
if nFilesOnly=="yes":
for i in builtNADicts:
if len(builtNADicts)==1:
suffix=""
else:
suffix="_%s" % ncount
nameparts=outfilenames[0].split(".")
newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1]
fileNames.append(newname)
ncount=ncount+1
return fileNames
msg="\n%s files to write" % len(builtNADicts)
print msg
outputMessage.append(msg)
count=1
ncount=1
for i in builtNADicts:
if len(outfilenames)==1:
if len(builtNADicts)==1:
suffix=""
else:
suffix="_%s" % ncount
nameparts=outfilenames[0].split(".")
newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1]
else:
newname=outfilenames[count-1]
msg="\nWriting output NASA Ames file: %s" % newname
print msg
outputMessage.append(msg)
builtNADict=i[0]
for key in naVars.keys():
if key in allowedOverwriteMetadata:
if key in arrayArgs:
newItem=naVars[key].split()
else:
newItem=naVars[key]
if newItem!=builtNADict[key]:
builtNADict[key]=newItem
msg="Metadata overwritten in output file: '%s' is now '%s'" % (key, builtNADict[key])
print msg
outputMessage.append(msg)
fileList=[]
# Cope with size limits if specified and FFI is 1001
if sizeLimit and (builtNADict["FFI"]==1001 and len(builtNADict["V"][0])>sizeLimit):
varList=builtNADict["V"]
arrayLength=len(varList[0])
nvolInfo=divmod(arrayLength, sizeLimit)
nvol=nvolInfo[0]
if nvolInfo[1]>0: nvol=nvol+1
start=0
letterCount=0
ivol=0
while start<arrayLength:
ivol=ivol+1
end=start+sizeLimit
if end>arrayLength:
end=arrayLength
currentBlock=[]
# Write new V array
for v in varList:
currentBlock.append(v[start:end])
# Adjust X accordingly
NADictCopy=modifyNADictCopy(builtNADict, currentBlock, start, end, ivol, nvol)
# Write data to output file
newnamePlusLetter="%s-%.3d.na" % (newname[:-3], ivol)
fileList.append(newnamePlusLetter)
general.openNAFile(newnamePlusLetter, 'w', NADictCopy, delimiter=delimiter, float_format=float_format)
msg="\nOutput files split on size limit: %s\nFilename used: %s" % (sizeLimit, newnamePlusLetter)
print msg
outputMessage.append(msg)
letterCount=letterCount+1
start=end
else:
general.openNAFile(newname, 'w', builtNADict, delimiter=delimiter, float_format=float_format)
msg="\nWrote the following variables:"+"\n\t"+("\n\t".join(i[1][0]))
print msg
outputMessage.append(msg)
if len(i[1][1])>0:
msg="\nWrote the following auxiliary variables:"
msg=msg+"\n\t"+("\n\t".join(i[1][1]))
if len(i[1][2])>0:
msg="\nWrote the following Singleton variables:"
msg=msg+"\n\t"+("\n\t".join(i[1][2]))
if len(fileList)>0:
msg=msg+("\n\nNASA Ames files written successfully: \n%s" % "\n".join(fileList))
count=count+len(fileList)
else:
msg=msg+"\n\nNASA Ames file written successfully: %s" % newname
count=count+1
ncount=ncount+1
print msg
outputMessage.append(msg)
if (count-1)==1:
plural=""
else:
plural="s"
msg="\n%s file%s written." % ((count-1), plural)
print msg
outputMessage.append(msg)
return outputMessage
def lonlatobs_to_lonlatmodel(WORKDIR, DIR_REGRID, lon_obs, lat_obs, lon_model,
lat_model, sst_nar):
#******************************************************************************
# Interpolation horizontale des observations sur la grille du modele
# lon_obs,lat_obs => lon_model,lat_model
#
# Utilisation du module SCRIP
#******************************************************************************
import os
import cdms, regrid
os.chdir(DIR_REGRID) # on se place dans le repertoire de travail
scrip_exec = '/export/home/logiciels/CDAT/installation/SCRIP/scrip'
scrip_in = """
&remap_inputs
num_maps = 2
grid1_file = '%s'
grid2_file = '%s'
interp_file1 = '%s'
interp_file2 = '%s'
map1_name = '%s Conservative Mapping'
map2_name = '%s Conservative Mapping'
map_method = 'conservative'
normalize_opt = 'frac'
output_opt = 'scrip'
restrict_type = 'latitude'
num_srch_bins = 90
luse_grid1_area = .false.
luse_grid2_area = .false.
/
"""
# ------------------------------------------------------------------------------------------
def toScripFormat(slab, file):
""" Dumps information about a slab into a file that can be used by the SCRIP regridder
Usage
toSCRIP(slab,file)
where:
file: is the output file to which information in SCRIP style will be dumped
slab: is the slab from which informations are gathered
"""
## Get definitions of grid
g = slab.getGrid()
print 'lat', g.getMesh()[0, 0]
print 'lon', g.getMesh()[0, 1]
m = slab.mask()
if m is not None:
print m.shape
g.setMask(m)
print g.shape
if g is None:
raise 'Error, no grid defined'
f = cdms.Cdunif.CdunifFile(file, 'w')
g.writeScrip(f, 'test_scrip')
f.close()
return
# -------------------------------------------------------------------------------------------
#-------------------------------------------------------
print ''
print '---------- INTERPOLATION HORIZONTALE ----------'
print ''
#-------------------------------------------------------
# Open the SCRIP remapping file and data file
direc = ''
fremap = cdms.open('rmp_NAR_to_MARS_conserv.nc')
# Input data array: sst_nar
infile = open(sys.argv[1])
for l in infile.xreadlines(): ## Loop thru input file
sp = l.split()
f = cdms.open(sp[0]) # Obs file
var = sp[1]
acc = sp[2] # Accronym for obs
finobs = acc + '_obs_SCRIP.nc'
s = f(var)
while s.rank() > 2: ## Gets only lat/lon
s = s[0]
f.close()
## Creates the obs SCRIP File
toScripFormat(s, finobs)
qry = 'select accro_orig, version_orig from modelversions,models where models.id=modelversions.model'
res = SQL.dbquery(qry)
for m in res:
## Prepares strings
rmp1 = 'rmp_NAR_to_MARS_conserv.nc'
rmp2 = 'rmp_MARS_to_NAR_conserv.nc'
tit1 = 'NAR to MARS'
tit2 = 'MARS to NAR'
fin = 'remap_grid_NAR.nc'
finobs = 'remap_grid_MARS.nc'
scrip_str = scrip_in % (fin, finobs, rmp1, rmp2, tit1, tit2)
dbpth = os.path.join(DB.root, 'grids', mod_accro)
rmp1db = os.path.join(dbpth, rmp1)
test = not os.path.exists(rmp1db)
test = True
if test:
print 'Creating remap files between %s and %s' % (mod_accro,
acc)
## Reads in grid file
fnm = os.path.join(dbpth, mod_accro + '_grid.nc')
f = cdms.open(fnm)
sm = f('mask')[0]
sm = MV.masked_equal(sm, 0)
toScripFormat(sm, fin)
fscr = open('scrip_in', 'w')
print >> fscr, scrip_str
fscr.close()
ln = os.popen(scrip_exec).readlines()
print 'Moving remap files'
ln = os.popen('mv %s %s' % (rmp1, rmp1db)).readlines()
print ln
os.popen('mv %s %s' %
(rmp2, os.path.join(dbpth, rmp2))).readlines()
os.remove('scrip_in')
os.remove(fin)
os.remove(finobs)
print 'Done'
## Bellow lines to remap using remap files
print 'Remapping using : ', rmp1
frmp1 = cdms.open(rmp1)
remapper = regrid.readRegridder(frmp1)
sst_nar_int2D = remapper(s)
## fout=cdms.open(acc+'_remapped_to_1x1.nc','w')
## fout.write(s1)
## fout.close()
# Read the SCRIP regridder
#regridf = regrid.readRegridder(fremap)
# Mettre dans une boucle sur les differents pas de temps
# Regrid the variable
#sst_nar_int2D = regridf(sst_nar)
return sst_nar_int2D
import MSU, cdms
weights_file = '../Data/weights.nc'
f = cdms.open(weights_file)
w = f(
'weights'
) # There's 3 channel on these weights => MSU computed for each one at once
f.close()
critw = 50. # Criteria to set to missing if too many missing values
file = '../Data/ta.nc'
fout = 'equivalent_msu.nc'
f = cdms.open(file)
ta = f('ta')
tc = ta.getTime().asComponentTime()
print 'Input data:', ta.shape
print 'Time span:', tc[0], tc[-1]
print 'Computing Equivalent MSU temperatures'
msu = MSU.msu(ta, w, critw)
print 'MSU computed, it has 3 channels as did the weights input'
msu.info()
fout = cdms.open(fout, 'w')
fout.write(msu[..., 0], typecode='f', id='tam2')
fout.write(msu[..., 1], typecode='f', id='tam4')
fout.write(msu[..., 2], typecode='f', id='tam6')
fout.close()
print 'Finished'
#!/usr/bin/env python
# Generates plots from an already existing theta_z.nc file
# Plots two types of isolines!!!
import cdms, vcs, cdutil, genutil, os, sys
from cdms import MV
xmin = 2000
xmax = 84000
zmin = 0
zmax = 20000
fid1 = cdms.open('./x-z_slices.nc')
# qc_z = fid1( 'qc_z' )
qc_z = fid1( 'qc_z', x=(xmin,xmax), z=(zmin,zmax) )
fid1.close()
fid2 = cdms.open('./m_eta_dot_l2_slice.nc')
m_eta_dot_z = fid2( 'm_eta_dot_l2_z', x=(xmin,xmax), z=(zmin,zmax) )
fid2.close()
# Number of time windows
ntm = len(m_eta_dot_z)
# Open canvas
canvas1 = vcs.init()
#!/usr/bin/env python
# Generates plots from an already existing theta_z.nc file
# Plots two types of isolines!!!
import cdms, vcs, cdutil, genutil, os, sys
from cdms import MV
xmin = 2000
xmax = 82000
zmin = 0
zmax = 20000
fid1 = cdms.open('./x-z_slices.nc')
qr_z = fid1('qr_z', x=(xmin, xmax), z=(zmin, zmax))
theta_z = fid1('theta_z', x=(xmin, xmax), z=(zmin, zmax))
eta_l2_z = fid1('eta_l2_z', x=(xmin, xmax), z=(zmin, zmax))
fid1.close()
# Number of time windows
ntm = len(theta_z)
# Open canvas
canvas1 = vcs.init()
# Create new templates from the existing 'ASD' template
t_asd = canvas1.createtemplate('new', 'ASD')
t_asd2 = canvas1.createtemplate('new2', 'ASD')
# Create new text orientation template objects for template t_asd
to1_xname = canvas1.createtextorientation('new1_xname', 'defcenter')
def writeOutput(infile, var, lat, lon, dav, dmax, dmin, sdupper, sdlower, location):
"""
Writes an output file of the variables generated.
"""
location=location.replace(" ","_").lower()
f=cdms.open(infile)
mapit={"apcp":("rainfall","l/m^2"), "tmpk":("temperature","K")}
varname=mapit[var][0]
units=mapit[var][1]
datetime=os.path.split(infile)[-1].split(".")[1]
outfile="%s_%s_%s.nc" % (datetime, location, varname)
outpath=os.path.split(infile)[0]
outfile=os.path.join(outpath, outfile)
fout=cdms.open(outfile, "w")
latax=cdms.createAxis([lat])
latax.units="degrees_north"
latax.id=latax.standard_name=latax.long_name="latitude"
lonax=cdms.createAxis([lon])
lonax.units="degrees_east"
lonax.id=lonax.standard_name=lonax.long_name="longitude"
tax=f[var].getTime() #f(var, level=slice(0,1), lat=slice(0,1), lon=slice(0,1)).getTime()
timeax=cdms.createAxis(Numeric.array(tax[0:tlen],'d'))
timeax.designateTime()
timeax.units=tax.units
#timeax.id=timeax.standard_name=timeax.long_name="time"
timeax.id="time"
timeax.title=tax.title
timeax.delta_t=tax.delta_t
timeax.init_time=tax.init_time
timeax.actual_range=tax.actual_range
del timeax.axis
del timeax.calendar
metadata=f[var]
fv=metadata.missing_value
newshape=(len(timeax), len(latax), len(lonax))
maxFound=20. # Set as our max value if not greater
for v in ((dav, "average"), (dmax, "maximum"), (dmin, "minimum"), \
(sdupper, "plus_std_dev"), (sdlower, "minus_std_dev"), ("always10", "always10")):
if type(v[0])==type("jlj") and v[0]=="always10":
print "Creating always equal to 10 variable."
always10=MA.zeros(newshape, 'f')+10.
#print always10.shape, dav.shape, type(dav)
newvar=cdms.createVariable(always10, axes=[timeax, latax, lonax], id=v[1], fill_value=fv)
newvar.longname="always10"
else:
data=v[0]
name=varname+"_"+v[1]
if not type(data)==type([1,2]):
data=data(squeeze=1)
data=MA.resize(data, newshape)
newvar=cdms.createVariable(data, axes=[timeax, latax, lonax], id=name, fill_value=fv)
newvar.long_name="%s - %s" % (varname.title(), v[1].replace("_", " "))
newvar.units=metadata.units
(dummy,vmax)=vcs.minmax(newvar)
if vmax>maxFound:
maxFound=vmax
fout.write(newvar)
fout.sync()
del newvar
fout.close()
return (outfile, varname, datetime, maxFound)
def get_ascii_ts(self, o1, rq, tslice='1931-1960', opt='decadal', mime='html', variable=None):
o1.write( 'starting get_ascii_ts\n' )
html = 'trying <br/>'
if not self.check_open():
return (1, 'failed to open file',' ','could not open %s<br/>' % self.ddd.this_file)
table_html = 'table not done'
data_display = 'data_display not done'
sel = { True:' selected="selected"', False:'' }
try:
nc = self.nc
vk = nc.variables.keys()
nsn = 0
for vv in vk:
if len( nc.variables[vv].shape ) == 3:
v = vv
standard_name = nc.variables[v].attributes.get('standard_name', nc.variables[v].attributes['name'] )
nsn += 1
if nsn != 1:
if nsn > 1:
return (0,' ', ' ','more than one named rank 3 variable in file')
else:
return (0,' ', ' ','no named rank 3 variable in file')
except:
rv = 1
return (rv,html,table_html, 'could not find suitable variable')
if nc.variables[v].attributes.has_key('standard_name'):
self.standard_name = standard_name
else:
self.standard_name = None
self.variable_name = v
if mime == 'csv':
self.make_csv_header()
try:
html += '%s<br/>' % standard_name
var = nc.variables[v]
data = nc.variables[v].getValue()
Axes = nc.variables[v].getAxisList()
fmt = fmts.get( v, '%s' )
tAx = nc.getAxis( 'time' )
yAx = nc.getAxis( 'latitude' )
xAx = nc.getAxis( 'longitude' )
yAxes = yAx.getValue()
xAxes = xAx.getValue()
tAxes = tAx.getValue()
##
## when submitted from the area selector, use those values,
## otherwise use the values from the hidden inputs, which are saved as lat and long,
## rather than indicies. This means we don't get shifted to a totally different domain
## when the model is switched.
##
if rq.fields.has_key( 'forwardbutton' ):
ix = iget( rq, 'IX', 0 )
iy = iget( rq, 'IY', 0 )
xtarg = xAxes[ix]
ytarg = yAxes[iy]
else:
yy = cget( rq, 'YY', '0.00' )
ytarg = float( yy )
dy = min( abs( yAxes - ytarg ) )
iy = 1
for i in range(len(yAxes)):
if abs( yAxes[i] - ytarg ) - dy < 0.001:
iy = i
xx = cget( rq, 'XX', '0.00' )
xtarg = float( xx )
dx = min( abs( xAxes - xtarg ) )
ix = 1
for i in range(len(xAxes)):
if abs( xAxes[i] - xtarg ) - dx < 0.001:
ix = i
if ix > len( xAxes ) -1:
ix = len( xAxes ) -1
if iy > len( yAxes ) -1:
iy = len( yAxes ) -1
html += 'zz %s, %s\n' % (str(ix), str(iy) )
if self.ddd.datasetid == 'cru21':
startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx,bounds='clim_bounds' )
else:
startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx,bounds='clim_bounds' )
## startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx )
trange = '%s-%s' % (startYear, endYear )
self.time_info = (startYear, startMonth, startDay, endYear)
########################################################################
############### region selection for html page #########################
########################################################################
if mime == 'html':
displayModes = ''
sel2 = { True:' disabled="disabled"', False:'' }
dm2 = '<center>Select display mode:<br/>'
dm = ['xy','ts']
dmn = ['Latitude-Longitude grid','Time series at a point']
for k in range(len(dm)):
dm2 += '<input type="submit"%s name="*%s" value="%s"/><br/>\n' % (sel2[k==1],dm[k],dmn[k])
dm2 += "</center>"
try:
dm = ['xy','ts']
dmn = ['Latitude-Longitude grid','Time series at a point']
for k in range(len(dm)):
displayModes += \
'<option value="%s"%s>Display as: %s</option>\n' % (k+1,sel[k==1],dmn[k])
except:
displayModes = 'zzzzzzzzzzzzzzzzzzzz'
x_options =' '
y_options =' '
html += '<br/>xxx'
for k in range(len(yAxes)):
y_options += \
'<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==iy],yAxes[k])
html += '<br/>xxx'
for k in range(len(xAxes)):
x_options += \
'<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==ix],xAxes[k])
######################################################################
html += 'xxx'
sl = [0,0,0]
islt = Axes.index( tAx )
isly = Axes.index( yAx )
islx = Axes.index( xAx )
html += 'xxx'
sl[islt] = slice( 0, len( tAx ) )
sl[isly] = iy
sl[islx] = ix
html += 'xxx'
x_label = 'Longitude'
y_label = 'Latitude'
data = nc.variables[v].getSlice( sl[0], sl[1], sl[2] )
html += '<br/>' + str( dir(tAx) )
html += '<br/>zzzzzzzzzzzzzzzzz<br/>' + str( dir(tAx.getExplicitBounds()) )
cpos = 'Position: %s, %s' % (ix,iy)
cpos = 'position: %7.2fE, %7.2fN' % (xAxes[ix],yAxes[iy])
if mime == 'csv':
self.csv_header_append( 'Attribute','g','Latitude','%7.2fN' % yAxes[iy] )
self.csv_header_append( 'Attribute','g','Longitude','%7.2fN' % xAxes[ix] )
## self.csv_header_append( 'Attribute','g','Start year', startYear )
## self.csv_header_append( 'Attribute','g','End year', endYear )
self.csv_header_append( 'Comment','First column','Time: days from start of year', ' ' )
self.csv_header_append( 'Comment','Second column','Month of year', ' ' )
self.csv_header_append( 'Comment','First row','Years averaged over', ' ' )
self.csv_header_append( 'Comment','Second row','Scenario', ' ' )
html += '<br/>xxx'
yname = axname( yAx, 'Y' )
xname = axname( xAx, 'X' )
tname = 'Time [%s]' % tAx.units
tname = 'Time [days from start of year]'
html += 'axes: %s, %s<br/>' % (xname,yname)
html += '<br/>xxx [checking datasetid %s]<br/>' % self.ddd.datasetid
if self.ddd.datasetid == 'ar4_gcm':
scenario = nc.getglobal( 'scenario_tag' )
model = nc.getglobal( 'model_tag' )
if scenario in ['COMMIT','SRA2','SRA1B','SRB1']:
slist = ['COMMIT','SRA2','SRA1B','SRB1']
trl = ['2011-2030','2046-2065','2080-2099']
elif scenario in ['1PTO2X','1PTO4X']:
slist = ['1PTO2X','1PTO4X' ]
if tslice in ['o0001-0030','o0031-0060','o0061-0090']:
trl = ['o0001-0030','o0031-0060','o0061-0090']
else:
trl = ['o0010-0039','o0046-0065','o0080-0099','o0180-0199']
else:
slist = [scenario]
if scenario == '20C3M':
trl = ['1901-1930','1931-1960','1961-1990']
elif scenario == 'PICTL':
if tslice in ['o0001-0030','o0031-0060','o0061-0090']:
trl = ['o0001-0030','o0031-0060','o0061-0090']
else:
trl = ['o0010-0039','o0046-0065','o0080-0099','o0180-0199']
elif self.ddd.datasetid == 'tar_gcm':
path = self.path
file = string.strip( string.split( path, '/' )[-1] )
bits = string.split( file, '_' )
scenario = bits[1]
model = bits[0]
slist = ['A1F','A1T','A1a','A2a','A2b','A2c','B1a','B2a','B2b']
trl = ['1980', '2020', '2050', '2080' ]
else:
trl = []
slist = ['Observations']
scenario = slist[0]
if opt == 'decadal':
for i in range(10):
trl.append( '%s-%s' % ( 1901 + 10*i, 1910 + 10*i ) )
else:
for i in range(3):
trl.append( '%s-%s' % ( 1901 + 30*i, 1930 + 30*i ) )
try:
import Numeric
thisZeros = Numeric.multiarray.zeros
except:
import numpy
thisZeros = numpy.zeros
lx = len(slist)*len(trl)
dv = thisZeros( (12,lx), 'f' )
iii = 0
mess = ''
for tr in trl:
html += '<br/> %s::%s ' % (tr,trange)
html += '<br/> %s ' % self.ddd.this_file
for s in slist:
html += '<br/> %s:: ' % s
if s == scenario and tr == tslice:
dv[:,iii] = data.tolist()
else:
if self.ddd.datasetid in ['tar_gcm', 'ar4_gcm']:
nf = string.replace( self.ddd.this_file, scenario, s )
else:
nf = self.ddd.this_file
if string.find( nf, tslice ) == -1:
o1.write( 'could not fine %s in %s\n' % (tslice, nf ) )
raise 'broken'
nf = string.replace( nf, tslice, tr )
if self.ddd.datasetid in ['tar_gcm']:
sln = string.lower(s)
nf = string.replace( nf, string.lower(scenario), sln )
if os.path.isfile( nf ) and os.stat(nf)[stat.ST_SIZE] > 128:
try:
import cdms
nc2 = cdms.open( nf, 'r' )
self.cdmsv = 1
except:
import cdms2
nc2 = cdms2.open( nf, 'r' )
self.cdmsv = 2
##import cdms
html += '<br/> ' + nf + '\n'
data2 = nc2.variables[v].getSlice( sl[0], sl[1], sl[2] )
mess += '<br/> %s %s %s' % (nf,data2[0],iii)
html += '<br/> %s %s %s' % (nf,data2[0],iii)
html += '<br/> %s %s' % (str(data2.shape), str(dv.shape))
html += 'z'
dv[:,iii] = data2.tolist()
nc2.close
html += 'z'
else:
o1.write( 'no data for %s\n' % nf )
dv[:,iii] = -999.
iii += 1
html += '<br/> data_dispay --- %s ' % cpos
html += '<br/> data_dispay --- %s ' % tname
html += '<br/> data_dispay --- %s ' % str(trl)
fmt = fmts.get( v, '%s' )
if standard_name in ['precipitation_flux_anomaly','precipitation_flux']:
units = 'mm/day'
factor = 3600*24
elif standard_name in ['precipitation_amount']:
units = 'mm/month'
factor = 1
else:
factor = 1
units = var.attributes['units']
if standard_name == 'air_temperature' and units == 'hPa':
units = 'Celsius'
if variable in utils.cru_vdict.keys():
display_name = utils.cru_vdict[variable]
elif variable in utils.tar_vdict.keys():
display_name = utils.tar_vdict[variable]
else:
display_name = standard_name
title = '%s [%s], %s' % (display_name,units,cpos)
html += '<br/>to utils.slice2table_1b .... '
(rv, data_display) = utils.slice2table_1b( title, tname, tAxes, 'Month', mnths, trl,slist, dv, fmt, factor, mime=mime )
html += 'back from utils.slice2table_1b<br/>\n'
###################
#### rv ==0 indicates failure, add comments to display.
if rv == 0:
data_display = html + data_display
if mime == 'html':
data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('YY',ytarg)
data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('XX',xtarg)
html += 'XXXX'
if mime == 'html':
o1.write( '--get_ascii_ts: reading template\n' )
html_pat = string.join( open( self.tmpl_xyt2, 'r' ).readlines() )
o1.write( '--get_ascii_ts: using template\n' )
table_html = html_pat % locals()
o1.write( '--get_ascii_ts: adding dm2\n' )
table_html += dm2
html += 'XXXX'
rv = 0
return (rv,html,table_html, data_display)
except:
rv = 1
return (rv,html,table_html, data_display)
#########################
# Simple netcdf plotter #
#########################
# Needed modules
import vcs, sys, cdms
# Arguments
if len(sys.argv) < 3:
print 'Usage: python quickview.py <filename> <varname>'
sys.exit(1)
filename = sys.argv[1]
varname = sys.argv[2]
# Open netcdf file
f = cdms.open(filename)
# Read our variable
s = f(varname)
# Create vcs canvas
x = vcs.init()
# Plot it
x.plot(s)
def simpleComp(fileA, fileB, tol=0.0):
'''A simple comparison function.
Attribute names and values are compared first.
Then the data arrays are compared within a
tolerance on a cell by cell basis.'''
# open files
fpA = cdms.open(fileA)
fpB = cdms.open(fileB)
# == global attributes ==
# compare attribute names
message = 'global attributes: '
globalNamesA = Set(fpA.listglobal())
globalNamesB = Set(fpB.listglobal())
symmetricDiff = globalNamesA ^ globalNamesB
if len(symmetricDiff) != 0:
(detailA, detailB) = setDiff(globalNamesA,globalNamesB,symmetricDiff)
return (FALSE,message,detailA,detailB)
# compare values
for globalName in globalNamesA:
# limit our checks to attributes with string values
if isinstance(eval(r'fpA.'+globalName),types.StringType):
globalValueA = eval(r'fpA.'+globalName)
globalValueB = eval(r'fpB.'+globalName)
if globalValueA != globalValueB:
message += globalName + ' values'
return(FALSE,message,globalValueA,globalValueB)
# == dimensions ==
# compare dimension names
dimNamesA = Set(fpA.listdimension())
dimNamesB = Set(fpB.listdimension())
symmetricDiff = dimNamesA ^ dimNamesB
if len(symmetricDiff) != 0:
message = 'dimensions:'
(detailA, detailB) = setDiff(dimNamesA,dimNamesB,symmetricDiff)
return (FALSE,message,detailA,detailB)
# loop over dimensions
for dimName in dimNamesA:
message = 'dimensions: '+ dimName
# compare attribute names
dimAttNamesA = Set(fpA[dimName].attributes.keys())
dimAttNamesB = Set(fpA[dimName].attributes.keys())
symmetricDiff = dimAttNamesA ^ dimAttNamesB
if len(symmetricDiff) != 0:
(detailA, detailB) = setDiff(dimAttNamesA,dimAttNamesB,symmetricDiff)
return (FALSE,message,detailA,detailB)
# compare attribute values
for dimAttName in dimAttNamesA:
# assuming objects we can compare
dimAttValueA = eval(r"fpA['"+dimName+r"']."+dimAttName)
dimAttValueB = eval(r"fpB['"+dimName+r"']."+dimAttName)
if dimAttValueA != dimAttValueB:
message += ': '+dimAttName
return (FALSE,message,dimAttValueA,dimAttValueB)
# compare data
dimDataShapeA = MV.shape(fpA[dimName])
dimDataShapeB = MV.shape(fpB[dimName])
if dimDataShapeA != dimDataShapeB:
message += ': data array shape'
return (FALSE,message,str(dimDataShapeA),str(dimDataShapeB))
maxDelta = MV.maximum(abs(fpA[dimName][:] - fpB[dimName][:]))
if maxDelta > tol:
message += ': delta: '+str(maxDelta)+' > '+str(tol)
return (FALSE,message,'','')
# == variables ==
# compare variable names
varNamesA = Set(fpA.listvariables())
varNamesB = Set(fpB.listvariables())
symmetricDiff = varNamesA ^ varNamesB
if len(symmetricDiff) != 0:
message = 'variables:'
(detailA, detailB) = setDiff(varNamesA,varNamesB,symmetricDiff)
return (FALSE,message,detailA,detailB)
# loop over variables
for varName in varNamesA:
message = 'variables: '+varName
# compare attribute names
varAttNamesA = Set(fpA[varName].attributes.keys())
varAttNamesB = Set(fpA[varName].attributes.keys())
symmetricDiff = varAttNamesA ^ varAttNamesB
if len(symmetricDiff) != 0:
(detailA, detailB) = setDiff(varAttNamesA,varAttNamesB,symmetricDiff)
return (FALSE,message,detailA,detailB)
# compare attribute values
for varAttName in varAttNamesA:
# assuming objects we can compare
varAttValueA = eval(r"fpA['"+varName+r"']."+varAttName)
varAttValueB = eval(r"fpB['"+varName+r"']."+varAttName)
if varAttValueA != varAttValueB:
message += ': '+varAttName
return (FALSE,message,varAttValueA,varAttValueB)
# compare data
varDataShapeA = MV.shape(fpA[varName])
varDataShapeB = MV.shape(fpB[varName])
if varDataShapeA != varDataShapeB:
message += ': data array shape'
return (FALSE,message,str(varDataShapeA),str(varDataShapeB))
maxDelta = MV.maximum(abs(fpA[varName][:] - fpB[varName][:]))
if maxDelta > tol:
message += ': delta: '+str(maxDelta)+' > '+str(tol)
return (FALSE,message,'','')
# close files
fpA.close()
fpB.close()
return (TRUE,'','','')
def lonlatobs_to_lonlatmodel(WORKDIR,DIR_REGRID,lon_obs,lat_obs,lon_model,lat_model,sst_nar):
#******************************************************************************
# Interpolation horizontale des observations sur la grille du modele
# lon_obs,lat_obs => lon_model,lat_model
#
# Utilisation du module SCRIP
#******************************************************************************
import os
import cdms, regrid
os.chdir(DIR_REGRID) # on se place dans le repertoire de travail
scrip_exec='/export/home/logiciels/CDAT/installation/SCRIP/scrip'
scrip_in="""
&remap_inputs
num_maps = 2
grid1_file = '%s'
grid2_file = '%s'
interp_file1 = '%s'
interp_file2 = '%s'
map1_name = '%s Conservative Mapping'
map2_name = '%s Conservative Mapping'
map_method = 'conservative'
normalize_opt = 'frac'
output_opt = 'scrip'
restrict_type = 'latitude'
num_srch_bins = 90
luse_grid1_area = .false.
luse_grid2_area = .false.
/
"""
# ------------------------------------------------------------------------------------------
def toScripFormat(slab,file):
""" Dumps information about a slab into a file that can be used by the SCRIP regridder
Usage
toSCRIP(slab,file)
where:
file: is the output file to which information in SCRIP style will be dumped
slab: is the slab from which informations are gathered
"""
## Get definitions of grid
g=slab.getGrid()
print 'lat',g.getMesh()[0,0]
print 'lon',g.getMesh()[0,1]
m=slab.mask()
if m is not None:
print m.shape
g.setMask(m)
print g.shape
if g is None:
raise 'Error, no grid defined'
f=cdms.Cdunif.CdunifFile(file,'w')
g.writeScrip(f,'test_scrip')
f.close()
return
# -------------------------------------------------------------------------------------------
#-------------------------------------------------------
print ''
print '---------- INTERPOLATION HORIZONTALE ----------'
print ''
#-------------------------------------------------------
# Open the SCRIP remapping file and data file
direc = ''
fremap = cdms.open('rmp_NAR_to_MARS_conserv.nc')
# Input data array: sst_nar
infile=open(sys.argv[1])
for l in infile.xreadlines(): ## Loop thru input file
sp=l.split()
f=cdms.open(sp[0]) # Obs file
var=sp[1]
acc=sp[2] # Accronym for obs
finobs=acc+'_obs_SCRIP.nc'
s=f(var)
while s.rank()>2: ## Gets only lat/lon
s=s[0]
f.close()
## Creates the obs SCRIP File
toScripFormat(s,finobs)
qry = 'select accro_orig, version_orig from modelversions,models where models.id=modelversions.model'
res=SQL.dbquery(qry)
for m in res:
## Prepares strings
rmp1='rmp_NAR_to_MARS_conserv.nc'
rmp2='rmp_MARS_to_NAR_conserv.nc'
tit1='NAR to MARS'
tit2='MARS to NAR'
fin='remap_grid_NAR.nc'
finobs='remap_grid_MARS.nc'
scrip_str=scrip_in % ( fin , finobs, rmp1, rmp2, tit1, tit2 )
dbpth=os.path.join(DB.root,'grids',mod_accro)
rmp1db=os.path.join(dbpth,rmp1)
test = not os.path.exists(rmp1db)
test = True
if test:
print 'Creating remap files between %s and %s' % (mod_accro,acc)
## Reads in grid file
fnm=os.path.join(dbpth,mod_accro+'_grid.nc')
f=cdms.open(fnm)
sm=f('mask')[0]
sm=MV.masked_equal(sm,0)
toScripFormat(sm,fin)
fscr=open('scrip_in','w')
print >> fscr,scrip_str
fscr.close()
ln=os.popen(scrip_exec).readlines()
print 'Moving remap files'
ln=os.popen('mv %s %s' % (rmp1,rmp1db)).readlines()
print ln
os.popen('mv %s %s' % (rmp2,os.path.join(dbpth,rmp2))).readlines()
os.remove('scrip_in')
os.remove(fin)
os.remove(finobs)
print 'Done'
## Bellow lines to remap using remap files
print 'Remapping using : ',rmp1
frmp1=cdms.open(rmp1)
remapper=regrid.readRegridder(frmp1)
sst_nar_int2D=remapper(s)
## fout=cdms.open(acc+'_remapped_to_1x1.nc','w')
## fout.write(s1)
## fout.close()
# Read the SCRIP regridder
#regridf = regrid.readRegridder(fremap)
# Mettre dans une boucle sur les differents pas de temps
# Regrid the variable
#sst_nar_int2D = regridf(sst_nar)
return sst_nar_int2D
import sys
#sys.path.insert(0,'../src/build/lib.linux-i686-2.4')
import cdms
import spanlib
import vcs
import MV
import Numeric
import cdutil
import genutil
cdms.axis.latitude_aliases.append('Y')
cdms.axis.longitude_aliases.append('X')
cdms.axis.time_aliases.append('T')
f=cdms.open('../example/data2.cdf')
s=f('ssta',time=slice(0,120))
print s.shape
SP=spanlib.SpAn(s)
eof,pc,ev = SP.pca()
print 'Done PCA, doing phases w/o mssa'
print 'Reconstructing no phases'
out = SP.reconstruct()
## phases = spanlib.phases(ffrec)
x=vcs.init()
# Needed modules
print 'Importing needed modules...'
# - base
import cdms,MV,vcs,sys,os
# - current version of spanlib is prioritary
if os.path.exists('../src/build/tmp_lib'):sys.path.insert(0,'../src/build/tmp_lib')
import spanlib
# We tell cdms that we have longitude, latitude and time
cdms.axis.latitude_aliases.append('Y')
cdms.axis.longitude_aliases.append('X')
cdms.axis.time_aliases.append('T')
# Simply open the netcdf file
print "Open file"
f=cdms.open('data2.cdf')
# Get our two datasets
print "Read two different regions"
s2=f('ssta',latitude=(-10,10),longitude=(110,180))
s1=f('ssta',latitude=(-15,15),longitude=(210,250))
s1[:,0:10,0:4]=MV.masked
s2[:,0:5,0:6]=MV.masked
print MV.average(s1.flat),MV.average(s2.flat)
# Stack the two dataset to have only one dataset
print "Stacking data"
res = spanlib.stackData(s1,s2)
# Create the analysis object
print "Creating SpAn object"
