def compVarSlice(fileA, fileB, var, dim, tol=0.0, start=0, end=0): '''Compare a slice (of given dimension) through named variable''' # open files fpA = cdms.open(fileA) fpB = cdms.open(fileB) # check named variable present in both files varsA = Set(fpA.listvariables()) varsB = Set(fpB.listvariables()) commonVars = varsA & varsB if var not in commonVars: fpA.close() fpB.close() return (FALSE, var + ' not common', varsA, varsB) # ditto for named dimension dimsA = Set(fpA.listdimension()) dimsB = Set(fpB.listdimension()) commonDims = dimsA & dimsB if dim not in commonDims: fpA.close() fpB.close() return (FALSE, dim + ' not common', dimsA, dimsB) # get the slices sliceA = eval(r"fpA('" + var + "'," + dim + "=slice(" + str(start) + "," + str(end) + "))") sliceB = eval(r"fpB('" + var + "'," + dim + "=slice(" + str(start) + "," + str(end) + "))") # close files fpA.close() fpB.close() # ensure dimensions of slices correct if sliceA.shape != sliceB.shape: return (FALSE, 'different shapes', sliceA.shape, sliceB.shape) if sliceA.shape[0] != end - start: return (FALSE, 'slice size wrong', str(sliceA.shape[0]), str(end - start)) if sliceA.shape[0] == 0: return (FALSE, 'slice size zero', str(sliceA.shape[0]), str(end - start)) # make actual comparison maxDelta = MV.maximum(abs(sliceA - sliceB)) if maxDelta > tol: return (FALSE, 'max diff > ' + str(tol), '', '') else: return (TRUE, '', '', '')
def _test(filename, varname=None): try: import cdms except ImportError: import cdms2 as cdms f = cdms.open(filename) if varname: var = f[varname] else: var = f.getVariables()[0] axes = var.getAxisList() rlons = axes[3] rlats = axes[2] lonBounds = rlons.getBounds() latBounds = rlats.getBounds() print "file longitudes" print lonBounds print "finding longitudes" for lon in range(360): print lon, findIndexLon(lon, lonBounds, useLimits=True) print "file latitudes" print latBounds print "finding latitudes" for lat in range(-90, 91): print lat, findIndex(lat, latBounds, useLimits=True)
def doRegrid(infile, varname, nodata, outfile, outformat, options, xRange, yRange, gt): # read infile a=cdms.open(infile) start_time=a.getAxis('time').asComponentTime()[] end_time=a.getAxis('time').asComponenentTime()[-1] data=a() if fh is None: exitMessage("Could not open file {0}. Exit 2.".format(infile), 2) print fh.history # read variable if varname not in fh.variables.keys(): exitMessage('variable named '+varname+' could not be found. Exit 4.', 4) yVar=numpy.array(fh.variables[varname][:]) # read lat lat=numpy.array(fh.variables['lat'][:]) # read lon lon=numpy.array(fh.variables['lon'][:]) # if arrays are 2-d, 1 lat/lon per pixel if len(lat.shape)==2: if (lat.shape[0]==yVar.shape[1]) and (lat.shape[1]==yVar.shape[2]): points = numpy.array( [numpy.ravel(lon), numpy.ravel(lat)] ).T elif (lat.shape[0]==yVar.shape[1]) and (lat.shape[1]==2): avgLon = numpy.repeat( numpy.array( [ numpy.ravel( 0.5*(lon[:,0]+lon[:,1]) ) ] ), lat.shape[0], 0) avgLat = numpy.repeat( numpy.array( [ numpy.ravel( 0.5*(lat[:,0]+lat[:,1]) ) ] ).T, lon.shape[0], 1) points = numpy.array( [numpy.ravel(avgLon), numpy.ravel(avgLat)] ).T else: exitMessage("Unknown case.", 2) elif len(lat.shape)==1: newLon = numpy.repeat( [ lon ] , lat.shape[0], 0) newLat = numpy.repeat( [ lat.T ] , lon.shape[0], 1) points = numpy.array( [numpy.ravel(newLon), numpy.ravel(newLat)] ).T else: exitMessage('Unknown structure for lat/lon. Exit(3).', 3) xi=[] for ix in xRange: for iy in yRange: xi.append([ix, iy]) # instantiate a file outDrv = gdal.GetDriverByName(outformat) outDS = outDrv.Create(outfile, len(xRange), len(yRange), yVar.shape[0], GDT_Float32, options) outDS.SetProjection(latlon()) outDS.SetGeoTransform(gt) # interpolate for iband in range(yVar.shape[0]): yInt = interpolate.griddata(points, yVar[iband, :, :].ravel(), xi, method='linear', fill_value=nodata) yIntToWrite = yInt.reshape(len(xRange), len(yRange)).copy() # save to file outDS.GetRasterBand( iband + 1 ).WriteArray( numpy.flipud(yIntToWrite.T), 0, 0) gdal.TermProgress_nocb( (iband+1)/float(yVar.shape[0]) ) gdal.TermProgress_nocb(1)
def compute(self) : args = inspect.getargspec(cdms.open) def_args = args[3] uri = None mode = def_args[0] template = def_args[1] dods = def_args[2] if not self.hasInputFromPort('uri') : print "Error: must have uri input" return if self.hasInputFromPort('uri') : inuri = self.getInputFromPort('uri') uri = os.path.join(sys.prefix, inuri) if self.hasInputFromPort('mode') : mode = self.getInputFromPort('mode') if self.hasInputFromPort('template') : template = self.getInputFromPort('template') if self.hasInputFromPort('dods') : dods = self.getInputFromPort('dods') # output the cdmsfile object. cdmsfile = cdms.open(uri,mode,template,dods) output = cdms_dataset(cdmsfile) self.setResult("cdms_dataset", output)
def compute(self): args = inspect.getargspec(cdms.open) def_args = args[3] uri = None mode = def_args[0] template = def_args[1] dods = def_args[2] if not self.has_input('uri'): print "Error: must have uri input" return if self.has_input('uri'): inuri = self.get_input('uri') uri = os.path.join(sys.prefix, inuri) if self.has_input('mode'): mode = self.get_input('mode') if self.has_input('template'): template = self.get_input('template') if self.has_input('dods'): dods = self.get_input('dods') # output the cdmsfile object. cdmsfile = cdms.open(uri, mode, template, dods) output = cdms_dataset(cdmsfile) self.set_output("cdms_dataset", output)
def compVarSlice(fileA, fileB, var, dim, tol=0.0, start=0, end=0): '''Compare a slice (of given dimension) through named variable''' # open files fpA = cdms.open(fileA) fpB = cdms.open(fileB) # check named variable present in both files varsA = Set(fpA.listvariables()) varsB = Set(fpB.listvariables()) commonVars = varsA & varsB if var not in commonVars: fpA.close() fpB.close() return (FALSE,var+' not common',varsA,varsB) # ditto for named dimension dimsA = Set(fpA.listdimension()) dimsB = Set(fpB.listdimension()) commonDims = dimsA & dimsB if dim not in commonDims: fpA.close() fpB.close() return (FALSE,dim+' not common',dimsA,dimsB) # get the slices sliceA = eval(r"fpA('"+var+"',"+dim+"=slice("+str(start)+","+str(end)+"))") sliceB = eval(r"fpB('"+var+"',"+dim+"=slice("+str(start)+","+str(end)+"))") # close files fpA.close() fpB.close() # ensure dimensions of slices correct if sliceA.shape != sliceB.shape: return (FALSE,'different shapes',sliceA.shape,sliceB.shape) if sliceA.shape[0] != end - start: return (FALSE,'slice size wrong',str(sliceA.shape[0]),str(end-start)) if sliceA.shape[0] == 0: return (FALSE,'slice size zero',str(sliceA.shape[0]),str(end-start)) # make actual comparison maxDelta = MV.maximum(abs(sliceA - sliceB)) if maxDelta > tol: return (FALSE,'max diff > '+str(tol),'','') else: return (TRUE,'','','')
def get_html(self): if self.ddd.filetype == 'nc': out = 'nc\n' try: import cdms self.cdmsv = 1 except: import cdms2 as cdms self.cdmsv = 2 try: nc = cdms.open( self.ddd.this_file, 'r' ) gl = nc.listglobal() gl.sort() stem = string.split( string.split( self.ddd.this_file, '/' )[-1], '.')[0] bits = string.split( stem, '_' ) out = '<h2>Global attributes</h2>\n<ul>\n' for g in gl: gv = nc.getglobal( g ) out += '<li>%s: %s</li>\n' % (g,str(gv)) out += '</ul><br/>\n\n' scenario = nc.getglobal( 'scenario_tag' ) model = nc.getglobal( 'model_tag' ) startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx, bounds='clim_bounds' ) trange = '%s-%s' % (startYear, endYear) self.time_info = (startYear, startMonth, startDay, endYear) vk = nc.variables.keys() vk.sort() for v in vk: Axes = map( lambda x: x.attributes.get('standard_name','-'), nc.variables[v].getAxisList() ) axesstring = string.join( Axes, ', ' ) out += '<h2>%s [ %s ]</h2>' % (v,axesstring) out += '<a href="/cgi-bin/ncbv/%s/%s">View data</a>\n' % (self.path,v) ak = nc.variables[v].attributes.keys() ak.sort() out += '<ul>\n' for a in ak: out += '<li>\n%s::%s\n</li>' % (a,nc.variables[v].attributes[a]) if len( nc.variables[v].shape ) == 3: standard_name = nc.variables[v].attributes.get('standard_name', nc.variables[v].attributes['name'] ) out += '</ul>\n' out += '<br/>%s, %s, %s, %s\n' % (standard_name, model, scenario, trange) nc.close() except: out += 'failed to construct html' raise return out
def scanFAAM(fileName=None, vars=None, nth=4, missingValuesToUse=(-9999., -32767.)): """ Scans every 'nth' variable in the list of variables (or found in the file and gets the first and last index of the first (time) dimension that holds real (non-missing) values. """ if type(missingValuesToUse) != type((1, 2)): missingValuesToUse = (missingValuesToUse, ) startList = [] endList = [] start = None end = None if not fileName and not vars: raise "You must provide either a file name or a list of cdms variables." if fileName: f = cdms.open(fileName) vars = f.listvariables() for var in vars: if type(var) != type(""): id = var.id else: id = var if id[-4:] == "FLAG" or id == "Time": continue if type(var) == type(""): var = f(var) step = 1000 while (start, end) == (None, None): (start, end) = findMissing(var, step, missingValuesToUse) step = step / 2 startList.append(start) endList.append(end) print "Start/End index: %s %s:%s" % (id, start, end) startMin = min(startList) endMax = max(endList) return (startMin, endMax)
def selectFeature(self, featureId, bbox, dimensionSpec): cacheKey = (featureId, self._makeDimKey(dimensionSpec)) try: ncfilename = self._map[cacheKey] except KeyError: (fd, ncfilename) = tempfile.mkstemp('.nc', featureId+'_', self._tempdir) os.close(fd) feature = self.dataset.getFeature(featureId) self._extract(feature, dimensionSpec, ncfilename) #self._map[cacheKey] = ncfilename d = cdms.open(ncfilename) (lon1, lat1, lon2, lat2) = bbox var = d(featureId, latitude=(lat1, lat2), longitude=(lon1, lon2), squeeze=1) d.close() return var
def check_open(self): if self.openned: return True try: file = self.ddd.this_file html = '%s <br/>' % file try: import cdms self.nc = cdms.open( file, 'r' ) self.cdmsv = 1 except: import cdms2 self.nc = cdms2.open( file, 'r' ) self.cdmsv = 2 self.openned = True return True except: return False
def scanFAAM(fileName=None, vars=None, nth=4, missingValuesToUse=(-9999., -32767.)): """ Scans every 'nth' variable in the list of variables (or found in the file and gets the first and last index of the first (time) dimension that holds real (non-missing) values. """ if type(missingValuesToUse)!=type((1,2)): missingValuesToUse=(missingValuesToUse,) startList=[] endList=[] start=None end=None if not fileName and not vars: raise "You must provide either a file name or a list of cdms variables." if fileName: f=cdms.open(fileName) vars=f.listvariables() for var in vars: if type(var)!=type(""): id=var.id else: id=var if id[-4:]=="FLAG" or id=="Time": continue if type(var)==type(""): var=f(var) step=1000 while (start, end)==(None, None): (start, end)=findMissing(var, step, missingValuesToUse) step=step/2 startList.append(start) endList.append(end) print "Start/End index: %s %s:%s" % (id, start, end) startMin=min(startList) endMax=max(endList) return (startMin, endMax)
def getVariable(infile, var, lat, lon): """Get the variable needed for the location (lat, lon).""" datafile=cdms.open(infile) if var==None: var=datafile.listvariables()[0] metadata=datafile[var] latax=metadata.getLatitude()[:] lonax=metadata.getLongitude()[:] lat=nudgeSingleValuesToAxisValues(lat, latax, "latitude")[0] lon=nudgeSingleValuesToAxisValues(lon, lonax, "longitude")[0] data=datafile(var, lat=lat, lon=lon, squeeze=1) print data.id data=data(time=slice(0,tlen)) data=accumulate24Hourly(data) datafile.close() # Do some working with the data depending on the variable if var=="tmpk": # ncep temperature data[:]=data[:]-270. data.units="degC" elif var=="apcp": # ncep rainfall # fix < zeros newarray=[] count=0 for i in data: newarray.append([]) for x in i: x=x[0] if x<0.0: x=0.0 newarray[count].append(x) count=count+1 data[:]=Numeric.array(newarray, 'f') return (data,lat,lon)
def processRainfall(file, outdir, var, north, west, south, east): "Subsets, averages, writes to binary files." f=cdms.open(file) v=f(var, lat=(south, north), lon=(west, east)) timevalues=v.getTime()[:] t0=timevalues[0] # I need to test if step 0 always has only missing values # remove -50 values??? v=MA.masked_less(v,0) # create average of all ensemble members av=MA.average(v, axis=1) # get stuff for name datetime=os.path.split(file)[-1].split(".")[1] outpaths=[] # now step through time dimension (0) count=0 for dslice in av: ts=timevalues[count]-t0 outfile="rainfall.%s.%dh.dat" % (datetime, ts) outpath=os.path.join(outdir, outfile) count=count+1 numarray=Numeric.array(dslice._data) sh=numarray.shape length=sh[0]*sh[1] flatarray=Numeric.resize(numarray, [length]) output=open(outpath, "wb") arr=array.array('f', flatarray) arr.tofile(output) output.close() print "Written:", outpath outpaths.append(outpath) return outpaths
import sys import cdms import spanlib import MV # sys.path.insert(0,'../src/build/lib.linux-i686-2.4') cdms.axis.latitude_aliases.append("Y") cdms.axis.longitude_aliases.append("X") cdms.axis.time_aliases.append("T") f = cdms.open("../example/data2.cdf") s2 = f("ssta", latitude=(-10, 10), longitude=(110, 180)) s1 = f("ssta", latitude=(-15, 15), longitude=(210, 250)) print "Data in:", s1.shape, s2.shape res = spanlib.stackData(s1, s2) print res[0].shape SP = spanlib.SpAn(MV.array(res[0]), weights=MV.array(res[1])) eof, pc, ev = SP.pca() ## for ax in res[3]: ## ax[0]=eof.getAxis(0)
import cdms, time, MA f = cdms.open('test0.nc', 'a') t = f.variables['air_temperature'] print time.time() x = t.getValue() for i in range(t.shape[0]): for j in range(t.shape[1]): x[i, j, :] += 2. t[:, :, :] = x print time.time() for i in range(t.shape[0]): for j in range(t.shape[1]): t[i, j, :] = MA.array(t[i, j, :] + 2., 'f') print time.time() f.close()
# Import the modules needed for the tutorial import vcs, cdms, cdutil, time, os, sys # Open data file: filepath = os.path.join(sys.prefix, 'sample_data/clt.nc') cdmsfile = cdms.open( filepath ) # Extract a 3 dimensional data set and get a subset of the time dimension data = cdmsfile('clt', longitude=(-180, 180), latitude = (-90., 90.)) # Initial VCS. v = vcs.init() # Show the list of persistent isofill graphics methods. v.show('isofill') # Assign the variable "cf_asd" to the persistent 'ASD' isofill graphics methods. cf_asd = v.getisofill( 'ASD' ) # Plot the data using the above boxfill graphics method. v.plot( data, cf_asd ) print "" print "Press the Return key to see next plot." sys.stdin.readline() # List the 'ASD' isofill graphics methods attributes. cf_asd.list() # change the isofill levels and the color indices. cf_asd.levels = ( [0,20],[20,40],[50,60],[60,70],[70,80],[90,100]) cf_asd.fillareacolors=( [22,44,66,88,110,132])
# import needed modules import cdms, vcs, cdutil, genutil, cdtime, MA, MV, sys, os # get an xml file or just a netcdf file file = os.path.join(sys.prefix,'sample_data/tas_ccsr-95a.xml') a=cdms.open(file) # get the data and print out it's shape data=a('tas') print data.shape #(12, 1, 32, 64) # # get the list of global attributes and put them into a dictionary list_file=a.attributes.keys() file_dic={} for i in range(0,len(list_file)): file_dic[i]=list_file[i],a.attributes[list_file[i] ] # see what is in a list and a dictionary print list_file print file_dic # now get the variable 'data' attributes and put into another dictionary list_data=data.attributes.keys() data_dic={} for i in range(0,len(list_data)): data_dic[i]=list_data[i],data.attributes[list_data[i] ] # print the list and the dictionary print list_data print data_dic
######################### # Simple netcdf plotter # ######################### # Needed modules import vcs, sys, cdms # Arguments if len(sys.argv) < 3: print 'Usage: python quickview.py <filename> <varname>' sys.exit(1) filename = sys.argv[1] varname = sys.argv[2] # Open netcdf file f=cdms.open(filename) # Read our variable s=f(varname) # Create vcs canvas x=vcs.init() # Plot it x.plot(s)
def nc_lon_cyc(ifile, ofile, verbose=0): ii = os.popen('cp %s %s' % (ifile, ofile), 'r') ii.readlines() ii.close() ii = os.popen('chmod 777 %s' % ofile, 'r') ii.readlines() ii.close() f = cdms.open(ofile, 'a') oldLonAxis = f.getAxis('longitude') if oldLonAxis.attributes.has_key('bounds'): oldLonBounds = oldLonAxis.attributes['bounds'] axisMappings, nShift, lonValShift = getNewLonAxis(oldLonAxis) nlon = len(oldLonAxis) if nlon != nShift * 2: print 'can only cope with shifts of half total length' print nlon, nShift raise 'config error' for k in range(len(oldLonAxis)): f.getAxis('longitude')[k] = axisMappings.getValue()[k] ## ## loop through variables and cycle on longitude axis. ## NB: only a limited number of variable shapes are supported. ## for var in f.getVariables(): oldLonAxis = var.getLongitude() if oldLonAxis: newAxes = var.getAxisList() if var.name == oldLonBounds: ## ## the cf-checker likes the bounds to include the coordinate value in ## an absolute sense, so that bounds for -180. should be, e.g., [-182, -178], ## NOT [178,-178]. ## if verbose > 0: print 'shifting bounds' print dir(var), var.getShape() ## ## shift longitude bounds by same amount as longitudes. ## va = var.getValue() for j in range(var.getShape()[0]): va[j, :] += lonValShift[j] var[:, :] = va if verbose > 1: print var.name, var index = newAxes.index(oldLonAxis) rank = var.rank() aa = var.getValue() if rank == 3 and index == 2: for i in range(nShift): var[:, :, i + nShift] = aa[:, :, i] var[:, :, i] = aa[:, :, i + nShift] elif rank == 2 and index == 0: for i in range(nShift): var[i + nShift, :] = aa[i, :] var[i, :] = aa[i + nShift, :] else: print rank, index raise 'rank index combination not programmed for' if verbose > 0: print var.name, index, rank f.history += '; longitude cycled to start at 180 degrees west' f.close()
#!/usr/bin/env python import cdms, vcs, cdutil, genutil, os, sys, Numeric from cdms import MV f_in1 = cdms.open('./pot_temp_geopotential.nc') f_in2 = cdms.open('./press_rho_temp.nc') print print "Reading in variables" theta = f_in1('theta') geopotential = f_in1('geopotential') pl = f_in2('pl') print print "Truncating variables" theta = cdms.createVariable(theta[:, :, 0, :], copy=1) geopotential = cdms.createVariable(geopotential[:, :, 0, :], copy=1) pl = cdms.createVariable(pl[:, :, 0, :], copy=1) print print "Calculating vpgf" exec(open('calc_vpgf.txt')) f_out = cdms.open('./vpgf.nc', 'w') print print "Writing vpgf to file" f_out.write(vpgf) print print "Done"
# Import modules import cdms, cdutil, MA, vcs, cdtime import string, Numeric, time, MV, sys, os from regrid import Regridder from genutil import statistics file1 = os.path.join(sys.prefix, 'sample_data/era40_tas_sample.nc') f1 = cdms.open( file1 ) f1.showvariable() f1.listdimension() print f1.getAxis('time').asComponentTime()[0] # 1990-1-1 0:0:0.0 print f1.getAxis('time').asComponentTime()[-1] # 1993-12-1 0:0:0.0 file2 = os.path.join(sys.prefix, 'sample_data/era15_tas_sample.nc') f2 = cdms.open( file2 ) f2.showvariable() f2.listdimension() print f2.getAxis('time').asComponentTime()[0] # 1989-1-1 0:0:0.0 print f2.getAxis('time').asComponentTime()[-1] # 1994-2-1 0:0:0.0 # get data with overlapping in overlapping time range data1 = f1('tas', time = ('1991-1-1','1993-12-1'))
import cdms from MA import allclose f = cdms.open('clt.nc') v = f('clt') s = cdms.selectors.timeslice(0,120,12) x = s.select(v) print x.shape y = v(s) print y.shape assert allclose(x,y) assert allclose(x, f('clt', time=slice(0,120,12))) x = v(s, longitude=(100,150)) print x.shape y = x(latitude=(-40,40)) print y.shape w = f('clt', cdms.selectors.longitude(100,140)) print w.shape print f('clt', slice(0,1), squeeze=1).shape
# # variable we want to work with var='tas' # # models we will be using model=['ccsr-95a', 'dnm-95a' ] # # set up a description string for addition to the global # attributes in the output netcdf # model_description='' # loop over all models--similar to a fortran do loop for i in range(0,len(model)): file_xml = os.path.join(sys.prefix,'sample_data/'+var+'_'+model[i]+'.xml') #a=cdms.open('/pcmdi/AMIP3/amip/mo/'+var+'/'+model[i]+'/'+var+'_'+model[i]+'.xml') a=cdms.open(file_xml) data=a[var] print '---- ', i, model[i],data.shape start_time = data.getTime().asComponentTime()[0] end_time = data.getTime().asComponentTime()[-1] print 'start time: ',start_time,' end time:',end_time time_len = len(data.getTime()) print 'time axis lenght: ', time_len a.close() dm=str(i)+' = '+model[i] model_description=model_description+', '+dm # print;print '__________________';print # set up an output array for the global time series glan=MA.zeros([len(model),time_len],MA.Float)
def simpleComp(fileA, fileB, tol=0.0): '''A simple comparison function. Attribute names and values are compared first. Then the data arrays are compared within a tolerance on a cell by cell basis.''' # open files fpA = cdms.open(fileA) fpB = cdms.open(fileB) # == global attributes == # compare attribute names message = 'global attributes: ' globalNamesA = Set(fpA.listglobal()) globalNamesB = Set(fpB.listglobal()) symmetricDiff = globalNamesA ^ globalNamesB if len(symmetricDiff) != 0: (detailA, detailB) = setDiff(globalNamesA, globalNamesB, symmetricDiff) return (FALSE, message, detailA, detailB) # compare values for globalName in globalNamesA: # limit our checks to attributes with string values if isinstance(eval(r'fpA.' + globalName), types.StringType): globalValueA = eval(r'fpA.' + globalName) globalValueB = eval(r'fpB.' + globalName) if globalValueA != globalValueB: message += globalName + ' values' return (FALSE, message, globalValueA, globalValueB) # == dimensions == # compare dimension names dimNamesA = Set(fpA.listdimension()) dimNamesB = Set(fpB.listdimension()) symmetricDiff = dimNamesA ^ dimNamesB if len(symmetricDiff) != 0: message = 'dimensions:' (detailA, detailB) = setDiff(dimNamesA, dimNamesB, symmetricDiff) return (FALSE, message, detailA, detailB) # loop over dimensions for dimName in dimNamesA: message = 'dimensions: ' + dimName # compare attribute names dimAttNamesA = Set(fpA[dimName].attributes.keys()) dimAttNamesB = Set(fpA[dimName].attributes.keys()) symmetricDiff = dimAttNamesA ^ dimAttNamesB if len(symmetricDiff) != 0: (detailA, detailB) = setDiff(dimAttNamesA, dimAttNamesB, symmetricDiff) return (FALSE, message, detailA, detailB) # compare attribute values for dimAttName in dimAttNamesA: # assuming objects we can compare dimAttValueA = eval(r"fpA['" + dimName + r"']." + dimAttName) dimAttValueB = eval(r"fpB['" + dimName + r"']." + dimAttName) if dimAttValueA != dimAttValueB: message += ': ' + dimAttName return (FALSE, message, dimAttValueA, dimAttValueB) # compare data dimDataShapeA = MV.shape(fpA[dimName]) dimDataShapeB = MV.shape(fpB[dimName]) if dimDataShapeA != dimDataShapeB: message += ': data array shape' return (FALSE, message, str(dimDataShapeA), str(dimDataShapeB)) maxDelta = MV.maximum(abs(fpA[dimName][:] - fpB[dimName][:])) if maxDelta > tol: message += ': delta: ' + str(maxDelta) + ' > ' + str(tol) return (FALSE, message, '', '') # == variables == # compare variable names varNamesA = Set(fpA.listvariables()) varNamesB = Set(fpB.listvariables()) symmetricDiff = varNamesA ^ varNamesB if len(symmetricDiff) != 0: message = 'variables:' (detailA, detailB) = setDiff(varNamesA, varNamesB, symmetricDiff) return (FALSE, message, detailA, detailB) # loop over variables for varName in varNamesA: message = 'variables: ' + varName # compare attribute names varAttNamesA = Set(fpA[varName].attributes.keys()) varAttNamesB = Set(fpA[varName].attributes.keys()) symmetricDiff = varAttNamesA ^ varAttNamesB if len(symmetricDiff) != 0: (detailA, detailB) = setDiff(varAttNamesA, varAttNamesB, symmetricDiff) return (FALSE, message, detailA, detailB) # compare attribute values for varAttName in varAttNamesA: # assuming objects we can compare varAttValueA = eval(r"fpA['" + varName + r"']." + varAttName) varAttValueB = eval(r"fpB['" + varName + r"']." + varAttName) if varAttValueA != varAttValueB: message += ': ' + varAttName return (FALSE, message, varAttValueA, varAttValueB) # compare data varDataShapeA = MV.shape(fpA[varName]) varDataShapeB = MV.shape(fpB[varName]) if varDataShapeA != varDataShapeB: message += ': data array shape' return (FALSE, message, str(varDataShapeA), str(varDataShapeB)) maxDelta = MV.maximum(abs(fpA[varName][:] - fpB[varName][:])) if maxDelta > tol: message += ': delta: ' + str(maxDelta) + ' > ' + str(tol) return (FALSE, message, '', '') # close files fpA.close() fpB.close() return (TRUE, '', '', '')
def extract_ij(ifile, ofile, i, j, verbose=0): oo = open(ofile, 'w') f = cdms.open(ifile, 'r') LonAxis = f.getAxis('longitude') LatAxis = f.getAxis('latitude') timeAxis = f.getAxis('time') time = f.getAxis('time').getValue() nlon = len(LonAxis) nlat = len(LatAxis) if i > nlon - 1: raise 'request is beyong end of longitude range' if j > nlat - 1: raise 'request is beyong end of latitude range' oo.write('"Time units","%s",,,,\n' % timeAxis.attributes['units']) oo.write('"Longitude",%s,%s,\n' % (LonAxis.getValue()[i], LonAxis.attributes['units'])) oo.write('"Latitude",%s,%s,\n' % (LatAxis.getValue()[j], LatAxis.attributes['units'])) if LonAxis.attributes.has_key('bounds'): LonBounds = LonAxis.attributes['bounds'] t = f.variables[LonBounds].getValue() oo.write('"Longitudinal bounds","%s",%s,%s,\n' % (LonBounds, t[i, 0], t[i, 1])) if LatAxis.attributes.has_key('bounds'): LatBounds = LatAxis.attributes['bounds'] t = f.variables[LatBounds].getValue() oo.write('"Latitudinal bounds","%s",%s,%s,\n' % (LatBounds, t[j, 0], t[j, 1])) ## ## loop through variables and cycle on longitude axis. ## NB: only a limited number of variable shapes are supported. ## for var in f.getVariables(): Lon = var.getLongitude() Lat = var.getLatitude() if Lat and Lon: Axes = var.getAxisList() ii = Axes.index(LonAxis) jj = Axes.index(LatAxis) rank = var.rank() aa = var.getValue() print var.shape oo.write('Name, "%s",\n' % (var.attributes['name'])) oo.write('Units, "%s",\n' % (var.attributes['units'])) oo.write('Standard name, "%s",\n' % (var.attributes['standard_name'])) if rank == 3 and ii == 1 and jj == 2: for k in range(len(time)): oo.write('%s, %s,\n' % (time[k], aa[k, i, j])) if rank == 3 and ii == 2 and jj == 1: for k in range(len(time)): print time[k], k, j, i print aa[k, j, i] oo.write('%s, %s, %s,\n' % (time[k], mn[k], aa[k, j, i])) else: print rank, ii, jj print 'rank index combination not programmed for' if verbose > 0: print var.name, index, rank f.close()
import MSU,cdms weights_file='../Data/weights.nc' f=cdms.open(weights_file) w=f('weights') # There's 3 channel on these weights => MSU computed for each one at once f.close() critw=50. # Criteria to set to missing if too many missing values file='../Data/ta.nc' fout='equivalent_msu.nc' f=cdms.open(file) ta=f('ta') tc=ta.getTime().asComponentTime() print 'Input data:',ta.shape print 'Time span:',tc[0],tc[-1] print 'Computing Equivalent MSU temperatures' msu=MSU.msu(ta,w,critw) print 'MSU computed, it has 3 channels as did the weights input' msu.info() fout=cdms.open(fout,'w') fout.write(msu[...,0],typecode='f',id='tam2') fout.write(msu[...,1],typecode='f',id='tam4') fout.write(msu[...,2],typecode='f',id='tam6') fout.close() print 'Finished'
def cdms2na(ncfile, na_file_names, naVars={}, variables=None, nFilesOnly="no", rule=None, ffi="automatic", delimiter=" ", float_format="%g", rules=None, sizeLimit=None): """ Main conversion function that calls the appropriate classes and functions to write a NASA Ames file. """ #print infilename, outfilenames, nFilesOnly, naVars, variables if type(na_file_names) == type("string"): na_file_names = [na_file_names] # Get which NASA Ames internal variables are allowed to be overwritten in the output files (i.e. by user inputs) allowedOverwriteMetadata = ("DATE", "RDATE", "ANAME", "MNAME", "ONAME", "ORG", "SNAME", "VNAME") arrayArgs=["DATE", "RDATE", "ANAME", "VNAME"] # ANAME[a] - array of 'a' x ANAME strings - aux var names # DATE (array of three) - UT date at which the data within the file starts # MNAME - mission name # ONAME - name of originator(s) # ORG - org or affiliation of originator(s) # RDATE (array of three) - date of data reduction or revision # SNAME - source of measurement or model output VNAME[n] - array of 'n' x # VNAME strings - var names. outputMessage=[] msg="Reading data from: %s\n" % infilename print msg outputMessage.append(msg) cdmsfile=cdms.open(infilename) globals=cdmsfile.attributes vars=[] if not variables: variables=cdmsfile.listvariables() #for var in cdmsfile.listvariables(): #vars.append(cdmsfile(var)) for variable in variables: varObj=cdmsfile(variable) # Deal with singleton variables if not hasattr(varObj, "rank"): varMetadata=cdmsfile[variable].attributes varValue=varObj #print varMetadata, varValue, varMetadata.keys(), varMetadata._obj_.id varObj=cdms.createVariable(Numeric.array(varObj), id=getBestName(varMetadata).replace(" ", "_"), attributes=varMetadata) #print varObj, dir(varObj); sys.exit() varObj.value=varObj._data[0] #varObj.rank=0 #print varObj, varObj.attributes vars.append(varObj) # Re-order variables if they have the attribute 'nasa_ames_var_number' orderedVars=[None]*1000 otherVars=[] for var in vars: varMetadata=cdmsfile[var] if hasattr(varMetadata, "nasa_ames_var_number"): num=varMetadata.nasa_ames_var_number orderedVars[num]=var else: otherVars.append(var) vars=[] for var in orderedVars: if var!=None: vars.append(var) vars=vars+otherVars builder=NAContentCollector(vars, globals, rule=rule, cdmsfile=cdmsfile) #print builder.na_dict["X"] builtNADicts=[[builder.na_dict, builder.varIDs]] if builder.varIDs==None: msg="\nNo files created after variables parsed." print msg outputMessage.append(msg) return outputMessage while len(builder.varBin)>0: builder=NAContentCollector(builder.varBin, globals, rule=rule, cdmsfile=cdmsfile) outputMessage=outputMessage+builder.outputMessage if builder.varIDs!=None: builtNADicts.append([builder.na_dict, builder.varIDs]) # Return only filenames if only want to know them now. ncount=1 fileNames=[] if nFilesOnly=="yes": for i in builtNADicts: if len(builtNADicts)==1: suffix="" else: suffix="_%s" % ncount nameparts=outfilenames[0].split(".") newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1] fileNames.append(newname) ncount=ncount+1 return fileNames msg="\n%s files to write" % len(builtNADicts) print msg outputMessage.append(msg) count=1 ncount=1 for i in builtNADicts: if len(outfilenames)==1: if len(builtNADicts)==1: suffix="" else: suffix="_%s" % ncount nameparts=outfilenames[0].split(".") newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1] else: newname=outfilenames[count-1] msg="\nWriting output NASA Ames file: %s" % newname print msg outputMessage.append(msg) builtNADict=i[0] for key in naVars.keys(): if key in allowedOverwriteMetadata: if key in arrayArgs: newItem=naVars[key].split() else: newItem=naVars[key] if newItem!=builtNADict[key]: builtNADict[key]=newItem msg="Metadata overwritten in output file: '%s' is now '%s'" % (key, builtNADict[key]) print msg outputMessage.append(msg) fileList=[] # Cope with size limits if specified and FFI is 1001 if sizeLimit and (builtNADict["FFI"]==1001 and len(builtNADict["V"][0])>sizeLimit): varList=builtNADict["V"] arrayLength=len(varList[0]) nvolInfo=divmod(arrayLength, sizeLimit) nvol=nvolInfo[0] if nvolInfo[1]>0: nvol=nvol+1 start=0 letterCount=0 ivol=0 while start<arrayLength: ivol=ivol+1 end=start+sizeLimit if end>arrayLength: end=arrayLength currentBlock=[] # Write new V array for v in varList: currentBlock.append(v[start:end]) # Adjust X accordingly NADictCopy=modifyNADictCopy(builtNADict, currentBlock, start, end, ivol, nvol) # Write data to output file newnamePlusLetter="%s-%.3d.na" % (newname[:-3], ivol) fileList.append(newnamePlusLetter) general.openNAFile(newnamePlusLetter, 'w', NADictCopy, delimiter=delimiter, float_format=float_format) msg="\nOutput files split on size limit: %s\nFilename used: %s" % (sizeLimit, newnamePlusLetter) print msg outputMessage.append(msg) letterCount=letterCount+1 start=end else: general.openNAFile(newname, 'w', builtNADict, delimiter=delimiter, float_format=float_format) msg="\nWrote the following variables:"+"\n\t"+("\n\t".join(i[1][0])) print msg outputMessage.append(msg) if len(i[1][1])>0: msg="\nWrote the following auxiliary variables:" msg=msg+"\n\t"+("\n\t".join(i[1][1])) if len(i[1][2])>0: msg="\nWrote the following Singleton variables:" msg=msg+"\n\t"+("\n\t".join(i[1][2])) if len(fileList)>0: msg=msg+("\n\nNASA Ames files written successfully: \n%s" % "\n".join(fileList)) count=count+len(fileList) else: msg=msg+"\n\nNASA Ames file written successfully: %s" % newname count=count+1 ncount=ncount+1 print msg outputMessage.append(msg) if (count-1)==1: plural="" else: plural="s" msg="\n%s file%s written." % ((count-1), plural) print msg outputMessage.append(msg) return outputMessage
def lonlatobs_to_lonlatmodel(WORKDIR, DIR_REGRID, lon_obs, lat_obs, lon_model, lat_model, sst_nar): #****************************************************************************** # Interpolation horizontale des observations sur la grille du modele # lon_obs,lat_obs => lon_model,lat_model # # Utilisation du module SCRIP #****************************************************************************** import os import cdms, regrid os.chdir(DIR_REGRID) # on se place dans le repertoire de travail scrip_exec = '/export/home/logiciels/CDAT/installation/SCRIP/scrip' scrip_in = """ &remap_inputs num_maps = 2 grid1_file = '%s' grid2_file = '%s' interp_file1 = '%s' interp_file2 = '%s' map1_name = '%s Conservative Mapping' map2_name = '%s Conservative Mapping' map_method = 'conservative' normalize_opt = 'frac' output_opt = 'scrip' restrict_type = 'latitude' num_srch_bins = 90 luse_grid1_area = .false. luse_grid2_area = .false. / """ # ------------------------------------------------------------------------------------------ def toScripFormat(slab, file): """ Dumps information about a slab into a file that can be used by the SCRIP regridder Usage toSCRIP(slab,file) where: file: is the output file to which information in SCRIP style will be dumped slab: is the slab from which informations are gathered """ ## Get definitions of grid g = slab.getGrid() print 'lat', g.getMesh()[0, 0] print 'lon', g.getMesh()[0, 1] m = slab.mask() if m is not None: print m.shape g.setMask(m) print g.shape if g is None: raise 'Error, no grid defined' f = cdms.Cdunif.CdunifFile(file, 'w') g.writeScrip(f, 'test_scrip') f.close() return # ------------------------------------------------------------------------------------------- #------------------------------------------------------- print '' print '---------- INTERPOLATION HORIZONTALE ----------' print '' #------------------------------------------------------- # Open the SCRIP remapping file and data file direc = '' fremap = cdms.open('rmp_NAR_to_MARS_conserv.nc') # Input data array: sst_nar infile = open(sys.argv[1]) for l in infile.xreadlines(): ## Loop thru input file sp = l.split() f = cdms.open(sp[0]) # Obs file var = sp[1] acc = sp[2] # Accronym for obs finobs = acc + '_obs_SCRIP.nc' s = f(var) while s.rank() > 2: ## Gets only lat/lon s = s[0] f.close() ## Creates the obs SCRIP File toScripFormat(s, finobs) qry = 'select accro_orig, version_orig from modelversions,models where models.id=modelversions.model' res = SQL.dbquery(qry) for m in res: ## Prepares strings rmp1 = 'rmp_NAR_to_MARS_conserv.nc' rmp2 = 'rmp_MARS_to_NAR_conserv.nc' tit1 = 'NAR to MARS' tit2 = 'MARS to NAR' fin = 'remap_grid_NAR.nc' finobs = 'remap_grid_MARS.nc' scrip_str = scrip_in % (fin, finobs, rmp1, rmp2, tit1, tit2) dbpth = os.path.join(DB.root, 'grids', mod_accro) rmp1db = os.path.join(dbpth, rmp1) test = not os.path.exists(rmp1db) test = True if test: print 'Creating remap files between %s and %s' % (mod_accro, acc) ## Reads in grid file fnm = os.path.join(dbpth, mod_accro + '_grid.nc') f = cdms.open(fnm) sm = f('mask')[0] sm = MV.masked_equal(sm, 0) toScripFormat(sm, fin) fscr = open('scrip_in', 'w') print >> fscr, scrip_str fscr.close() ln = os.popen(scrip_exec).readlines() print 'Moving remap files' ln = os.popen('mv %s %s' % (rmp1, rmp1db)).readlines() print ln os.popen('mv %s %s' % (rmp2, os.path.join(dbpth, rmp2))).readlines() os.remove('scrip_in') os.remove(fin) os.remove(finobs) print 'Done' ## Bellow lines to remap using remap files print 'Remapping using : ', rmp1 frmp1 = cdms.open(rmp1) remapper = regrid.readRegridder(frmp1) sst_nar_int2D = remapper(s) ## fout=cdms.open(acc+'_remapped_to_1x1.nc','w') ## fout.write(s1) ## fout.close() # Read the SCRIP regridder #regridf = regrid.readRegridder(fremap) # Mettre dans une boucle sur les differents pas de temps # Regrid the variable #sst_nar_int2D = regridf(sst_nar) return sst_nar_int2D
import MSU, cdms weights_file = '../Data/weights.nc' f = cdms.open(weights_file) w = f( 'weights' ) # There's 3 channel on these weights => MSU computed for each one at once f.close() critw = 50. # Criteria to set to missing if too many missing values file = '../Data/ta.nc' fout = 'equivalent_msu.nc' f = cdms.open(file) ta = f('ta') tc = ta.getTime().asComponentTime() print 'Input data:', ta.shape print 'Time span:', tc[0], tc[-1] print 'Computing Equivalent MSU temperatures' msu = MSU.msu(ta, w, critw) print 'MSU computed, it has 3 channels as did the weights input' msu.info() fout = cdms.open(fout, 'w') fout.write(msu[..., 0], typecode='f', id='tam2') fout.write(msu[..., 1], typecode='f', id='tam4') fout.write(msu[..., 2], typecode='f', id='tam6') fout.close() print 'Finished'
#!/usr/bin/env python # Generates plots from an already existing theta_z.nc file # Plots two types of isolines!!! import cdms, vcs, cdutil, genutil, os, sys from cdms import MV xmin = 2000 xmax = 84000 zmin = 0 zmax = 20000 fid1 = cdms.open('./x-z_slices.nc') # qc_z = fid1( 'qc_z' ) qc_z = fid1( 'qc_z', x=(xmin,xmax), z=(zmin,zmax) ) fid1.close() fid2 = cdms.open('./m_eta_dot_l2_slice.nc') m_eta_dot_z = fid2( 'm_eta_dot_l2_z', x=(xmin,xmax), z=(zmin,zmax) ) fid2.close() # Number of time windows ntm = len(m_eta_dot_z) # Open canvas canvas1 = vcs.init()
#!/usr/bin/env python # Generates plots from an already existing theta_z.nc file # Plots two types of isolines!!! import cdms, vcs, cdutil, genutil, os, sys from cdms import MV xmin = 2000 xmax = 82000 zmin = 0 zmax = 20000 fid1 = cdms.open('./x-z_slices.nc') qr_z = fid1('qr_z', x=(xmin, xmax), z=(zmin, zmax)) theta_z = fid1('theta_z', x=(xmin, xmax), z=(zmin, zmax)) eta_l2_z = fid1('eta_l2_z', x=(xmin, xmax), z=(zmin, zmax)) fid1.close() # Number of time windows ntm = len(theta_z) # Open canvas canvas1 = vcs.init() # Create new templates from the existing 'ASD' template t_asd = canvas1.createtemplate('new', 'ASD') t_asd2 = canvas1.createtemplate('new2', 'ASD') # Create new text orientation template objects for template t_asd to1_xname = canvas1.createtextorientation('new1_xname', 'defcenter')
def writeOutput(infile, var, lat, lon, dav, dmax, dmin, sdupper, sdlower, location): """ Writes an output file of the variables generated. """ location=location.replace(" ","_").lower() f=cdms.open(infile) mapit={"apcp":("rainfall","l/m^2"), "tmpk":("temperature","K")} varname=mapit[var][0] units=mapit[var][1] datetime=os.path.split(infile)[-1].split(".")[1] outfile="%s_%s_%s.nc" % (datetime, location, varname) outpath=os.path.split(infile)[0] outfile=os.path.join(outpath, outfile) fout=cdms.open(outfile, "w") latax=cdms.createAxis([lat]) latax.units="degrees_north" latax.id=latax.standard_name=latax.long_name="latitude" lonax=cdms.createAxis([lon]) lonax.units="degrees_east" lonax.id=lonax.standard_name=lonax.long_name="longitude" tax=f[var].getTime() #f(var, level=slice(0,1), lat=slice(0,1), lon=slice(0,1)).getTime() timeax=cdms.createAxis(Numeric.array(tax[0:tlen],'d')) timeax.designateTime() timeax.units=tax.units #timeax.id=timeax.standard_name=timeax.long_name="time" timeax.id="time" timeax.title=tax.title timeax.delta_t=tax.delta_t timeax.init_time=tax.init_time timeax.actual_range=tax.actual_range del timeax.axis del timeax.calendar metadata=f[var] fv=metadata.missing_value newshape=(len(timeax), len(latax), len(lonax)) maxFound=20. # Set as our max value if not greater for v in ((dav, "average"), (dmax, "maximum"), (dmin, "minimum"), \ (sdupper, "plus_std_dev"), (sdlower, "minus_std_dev"), ("always10", "always10")): if type(v[0])==type("jlj") and v[0]=="always10": print "Creating always equal to 10 variable." always10=MA.zeros(newshape, 'f')+10. #print always10.shape, dav.shape, type(dav) newvar=cdms.createVariable(always10, axes=[timeax, latax, lonax], id=v[1], fill_value=fv) newvar.longname="always10" else: data=v[0] name=varname+"_"+v[1] if not type(data)==type([1,2]): data=data(squeeze=1) data=MA.resize(data, newshape) newvar=cdms.createVariable(data, axes=[timeax, latax, lonax], id=name, fill_value=fv) newvar.long_name="%s - %s" % (varname.title(), v[1].replace("_", " ")) newvar.units=metadata.units (dummy,vmax)=vcs.minmax(newvar) if vmax>maxFound: maxFound=vmax fout.write(newvar) fout.sync() del newvar fout.close() return (outfile, varname, datetime, maxFound)
def get_ascii_ts(self, o1, rq, tslice='1931-1960', opt='decadal', mime='html', variable=None): o1.write( 'starting get_ascii_ts\n' ) html = 'trying <br/>' if not self.check_open(): return (1, 'failed to open file',' ','could not open %s<br/>' % self.ddd.this_file) table_html = 'table not done' data_display = 'data_display not done' sel = { True:' selected="selected"', False:'' } try: nc = self.nc vk = nc.variables.keys() nsn = 0 for vv in vk: if len( nc.variables[vv].shape ) == 3: v = vv standard_name = nc.variables[v].attributes.get('standard_name', nc.variables[v].attributes['name'] ) nsn += 1 if nsn != 1: if nsn > 1: return (0,' ', ' ','more than one named rank 3 variable in file') else: return (0,' ', ' ','no named rank 3 variable in file') except: rv = 1 return (rv,html,table_html, 'could not find suitable variable') if nc.variables[v].attributes.has_key('standard_name'): self.standard_name = standard_name else: self.standard_name = None self.variable_name = v if mime == 'csv': self.make_csv_header() try: html += '%s<br/>' % standard_name var = nc.variables[v] data = nc.variables[v].getValue() Axes = nc.variables[v].getAxisList() fmt = fmts.get( v, '%s' ) tAx = nc.getAxis( 'time' ) yAx = nc.getAxis( 'latitude' ) xAx = nc.getAxis( 'longitude' ) yAxes = yAx.getValue() xAxes = xAx.getValue() tAxes = tAx.getValue() ## ## when submitted from the area selector, use those values, ## otherwise use the values from the hidden inputs, which are saved as lat and long, ## rather than indicies. This means we don't get shifted to a totally different domain ## when the model is switched. ## if rq.fields.has_key( 'forwardbutton' ): ix = iget( rq, 'IX', 0 ) iy = iget( rq, 'IY', 0 ) xtarg = xAxes[ix] ytarg = yAxes[iy] else: yy = cget( rq, 'YY', '0.00' ) ytarg = float( yy ) dy = min( abs( yAxes - ytarg ) ) iy = 1 for i in range(len(yAxes)): if abs( yAxes[i] - ytarg ) - dy < 0.001: iy = i xx = cget( rq, 'XX', '0.00' ) xtarg = float( xx ) dx = min( abs( xAxes - xtarg ) ) ix = 1 for i in range(len(xAxes)): if abs( xAxes[i] - xtarg ) - dx < 0.001: ix = i if ix > len( xAxes ) -1: ix = len( xAxes ) -1 if iy > len( yAxes ) -1: iy = len( yAxes ) -1 html += 'zz %s, %s\n' % (str(ix), str(iy) ) if self.ddd.datasetid == 'cru21': startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx,bounds='clim_bounds' ) else: startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx,bounds='clim_bounds' ) ## startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx ) trange = '%s-%s' % (startYear, endYear ) self.time_info = (startYear, startMonth, startDay, endYear) ######################################################################## ############### region selection for html page ######################### ######################################################################## if mime == 'html': displayModes = '' sel2 = { True:' disabled="disabled"', False:'' } dm2 = '<center>Select display mode:<br/>' dm = ['xy','ts'] dmn = ['Latitude-Longitude grid','Time series at a point'] for k in range(len(dm)): dm2 += '<input type="submit"%s name="*%s" value="%s"/><br/>\n' % (sel2[k==1],dm[k],dmn[k]) dm2 += "</center>" try: dm = ['xy','ts'] dmn = ['Latitude-Longitude grid','Time series at a point'] for k in range(len(dm)): displayModes += \ '<option value="%s"%s>Display as: %s</option>\n' % (k+1,sel[k==1],dmn[k]) except: displayModes = 'zzzzzzzzzzzzzzzzzzzz' x_options =' ' y_options =' ' html += '<br/>xxx' for k in range(len(yAxes)): y_options += \ '<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==iy],yAxes[k]) html += '<br/>xxx' for k in range(len(xAxes)): x_options += \ '<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==ix],xAxes[k]) ###################################################################### html += 'xxx' sl = [0,0,0] islt = Axes.index( tAx ) isly = Axes.index( yAx ) islx = Axes.index( xAx ) html += 'xxx' sl[islt] = slice( 0, len( tAx ) ) sl[isly] = iy sl[islx] = ix html += 'xxx' x_label = 'Longitude' y_label = 'Latitude' data = nc.variables[v].getSlice( sl[0], sl[1], sl[2] ) html += '<br/>' + str( dir(tAx) ) html += '<br/>zzzzzzzzzzzzzzzzz<br/>' + str( dir(tAx.getExplicitBounds()) ) cpos = 'Position: %s, %s' % (ix,iy) cpos = 'position: %7.2fE, %7.2fN' % (xAxes[ix],yAxes[iy]) if mime == 'csv': self.csv_header_append( 'Attribute','g','Latitude','%7.2fN' % yAxes[iy] ) self.csv_header_append( 'Attribute','g','Longitude','%7.2fN' % xAxes[ix] ) ## self.csv_header_append( 'Attribute','g','Start year', startYear ) ## self.csv_header_append( 'Attribute','g','End year', endYear ) self.csv_header_append( 'Comment','First column','Time: days from start of year', ' ' ) self.csv_header_append( 'Comment','Second column','Month of year', ' ' ) self.csv_header_append( 'Comment','First row','Years averaged over', ' ' ) self.csv_header_append( 'Comment','Second row','Scenario', ' ' ) html += '<br/>xxx' yname = axname( yAx, 'Y' ) xname = axname( xAx, 'X' ) tname = 'Time [%s]' % tAx.units tname = 'Time [days from start of year]' html += 'axes: %s, %s<br/>' % (xname,yname) html += '<br/>xxx [checking datasetid %s]<br/>' % self.ddd.datasetid if self.ddd.datasetid == 'ar4_gcm': scenario = nc.getglobal( 'scenario_tag' ) model = nc.getglobal( 'model_tag' ) if scenario in ['COMMIT','SRA2','SRA1B','SRB1']: slist = ['COMMIT','SRA2','SRA1B','SRB1'] trl = ['2011-2030','2046-2065','2080-2099'] elif scenario in ['1PTO2X','1PTO4X']: slist = ['1PTO2X','1PTO4X' ] if tslice in ['o0001-0030','o0031-0060','o0061-0090']: trl = ['o0001-0030','o0031-0060','o0061-0090'] else: trl = ['o0010-0039','o0046-0065','o0080-0099','o0180-0199'] else: slist = [scenario] if scenario == '20C3M': trl = ['1901-1930','1931-1960','1961-1990'] elif scenario == 'PICTL': if tslice in ['o0001-0030','o0031-0060','o0061-0090']: trl = ['o0001-0030','o0031-0060','o0061-0090'] else: trl = ['o0010-0039','o0046-0065','o0080-0099','o0180-0199'] elif self.ddd.datasetid == 'tar_gcm': path = self.path file = string.strip( string.split( path, '/' )[-1] ) bits = string.split( file, '_' ) scenario = bits[1] model = bits[0] slist = ['A1F','A1T','A1a','A2a','A2b','A2c','B1a','B2a','B2b'] trl = ['1980', '2020', '2050', '2080' ] else: trl = [] slist = ['Observations'] scenario = slist[0] if opt == 'decadal': for i in range(10): trl.append( '%s-%s' % ( 1901 + 10*i, 1910 + 10*i ) ) else: for i in range(3): trl.append( '%s-%s' % ( 1901 + 30*i, 1930 + 30*i ) ) try: import Numeric thisZeros = Numeric.multiarray.zeros except: import numpy thisZeros = numpy.zeros lx = len(slist)*len(trl) dv = thisZeros( (12,lx), 'f' ) iii = 0 mess = '' for tr in trl: html += '<br/> %s::%s ' % (tr,trange) html += '<br/> %s ' % self.ddd.this_file for s in slist: html += '<br/> %s:: ' % s if s == scenario and tr == tslice: dv[:,iii] = data.tolist() else: if self.ddd.datasetid in ['tar_gcm', 'ar4_gcm']: nf = string.replace( self.ddd.this_file, scenario, s ) else: nf = self.ddd.this_file if string.find( nf, tslice ) == -1: o1.write( 'could not fine %s in %s\n' % (tslice, nf ) ) raise 'broken' nf = string.replace( nf, tslice, tr ) if self.ddd.datasetid in ['tar_gcm']: sln = string.lower(s) nf = string.replace( nf, string.lower(scenario), sln ) if os.path.isfile( nf ) and os.stat(nf)[stat.ST_SIZE] > 128: try: import cdms nc2 = cdms.open( nf, 'r' ) self.cdmsv = 1 except: import cdms2 nc2 = cdms2.open( nf, 'r' ) self.cdmsv = 2 ##import cdms html += '<br/> ' + nf + '\n' data2 = nc2.variables[v].getSlice( sl[0], sl[1], sl[2] ) mess += '<br/> %s %s %s' % (nf,data2[0],iii) html += '<br/> %s %s %s' % (nf,data2[0],iii) html += '<br/> %s %s' % (str(data2.shape), str(dv.shape)) html += 'z' dv[:,iii] = data2.tolist() nc2.close html += 'z' else: o1.write( 'no data for %s\n' % nf ) dv[:,iii] = -999. iii += 1 html += '<br/> data_dispay --- %s ' % cpos html += '<br/> data_dispay --- %s ' % tname html += '<br/> data_dispay --- %s ' % str(trl) fmt = fmts.get( v, '%s' ) if standard_name in ['precipitation_flux_anomaly','precipitation_flux']: units = 'mm/day' factor = 3600*24 elif standard_name in ['precipitation_amount']: units = 'mm/month' factor = 1 else: factor = 1 units = var.attributes['units'] if standard_name == 'air_temperature' and units == 'hPa': units = 'Celsius' if variable in utils.cru_vdict.keys(): display_name = utils.cru_vdict[variable] elif variable in utils.tar_vdict.keys(): display_name = utils.tar_vdict[variable] else: display_name = standard_name title = '%s [%s], %s' % (display_name,units,cpos) html += '<br/>to utils.slice2table_1b .... ' (rv, data_display) = utils.slice2table_1b( title, tname, tAxes, 'Month', mnths, trl,slist, dv, fmt, factor, mime=mime ) html += 'back from utils.slice2table_1b<br/>\n' ################### #### rv ==0 indicates failure, add comments to display. if rv == 0: data_display = html + data_display if mime == 'html': data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('YY',ytarg) data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('XX',xtarg) html += 'XXXX' if mime == 'html': o1.write( '--get_ascii_ts: reading template\n' ) html_pat = string.join( open( self.tmpl_xyt2, 'r' ).readlines() ) o1.write( '--get_ascii_ts: using template\n' ) table_html = html_pat % locals() o1.write( '--get_ascii_ts: adding dm2\n' ) table_html += dm2 html += 'XXXX' rv = 0 return (rv,html,table_html, data_display) except: rv = 1 return (rv,html,table_html, data_display)
######################### # Simple netcdf plotter # ######################### # Needed modules import vcs, sys, cdms # Arguments if len(sys.argv) < 3: print 'Usage: python quickview.py <filename> <varname>' sys.exit(1) filename = sys.argv[1] varname = sys.argv[2] # Open netcdf file f = cdms.open(filename) # Read our variable s = f(varname) # Create vcs canvas x = vcs.init() # Plot it x.plot(s)
def simpleComp(fileA, fileB, tol=0.0): '''A simple comparison function. Attribute names and values are compared first. Then the data arrays are compared within a tolerance on a cell by cell basis.''' # open files fpA = cdms.open(fileA) fpB = cdms.open(fileB) # == global attributes == # compare attribute names message = 'global attributes: ' globalNamesA = Set(fpA.listglobal()) globalNamesB = Set(fpB.listglobal()) symmetricDiff = globalNamesA ^ globalNamesB if len(symmetricDiff) != 0: (detailA, detailB) = setDiff(globalNamesA,globalNamesB,symmetricDiff) return (FALSE,message,detailA,detailB) # compare values for globalName in globalNamesA: # limit our checks to attributes with string values if isinstance(eval(r'fpA.'+globalName),types.StringType): globalValueA = eval(r'fpA.'+globalName) globalValueB = eval(r'fpB.'+globalName) if globalValueA != globalValueB: message += globalName + ' values' return(FALSE,message,globalValueA,globalValueB) # == dimensions == # compare dimension names dimNamesA = Set(fpA.listdimension()) dimNamesB = Set(fpB.listdimension()) symmetricDiff = dimNamesA ^ dimNamesB if len(symmetricDiff) != 0: message = 'dimensions:' (detailA, detailB) = setDiff(dimNamesA,dimNamesB,symmetricDiff) return (FALSE,message,detailA,detailB) # loop over dimensions for dimName in dimNamesA: message = 'dimensions: '+ dimName # compare attribute names dimAttNamesA = Set(fpA[dimName].attributes.keys()) dimAttNamesB = Set(fpA[dimName].attributes.keys()) symmetricDiff = dimAttNamesA ^ dimAttNamesB if len(symmetricDiff) != 0: (detailA, detailB) = setDiff(dimAttNamesA,dimAttNamesB,symmetricDiff) return (FALSE,message,detailA,detailB) # compare attribute values for dimAttName in dimAttNamesA: # assuming objects we can compare dimAttValueA = eval(r"fpA['"+dimName+r"']."+dimAttName) dimAttValueB = eval(r"fpB['"+dimName+r"']."+dimAttName) if dimAttValueA != dimAttValueB: message += ': '+dimAttName return (FALSE,message,dimAttValueA,dimAttValueB) # compare data dimDataShapeA = MV.shape(fpA[dimName]) dimDataShapeB = MV.shape(fpB[dimName]) if dimDataShapeA != dimDataShapeB: message += ': data array shape' return (FALSE,message,str(dimDataShapeA),str(dimDataShapeB)) maxDelta = MV.maximum(abs(fpA[dimName][:] - fpB[dimName][:])) if maxDelta > tol: message += ': delta: '+str(maxDelta)+' > '+str(tol) return (FALSE,message,'','') # == variables == # compare variable names varNamesA = Set(fpA.listvariables()) varNamesB = Set(fpB.listvariables()) symmetricDiff = varNamesA ^ varNamesB if len(symmetricDiff) != 0: message = 'variables:' (detailA, detailB) = setDiff(varNamesA,varNamesB,symmetricDiff) return (FALSE,message,detailA,detailB) # loop over variables for varName in varNamesA: message = 'variables: '+varName # compare attribute names varAttNamesA = Set(fpA[varName].attributes.keys()) varAttNamesB = Set(fpA[varName].attributes.keys()) symmetricDiff = varAttNamesA ^ varAttNamesB if len(symmetricDiff) != 0: (detailA, detailB) = setDiff(varAttNamesA,varAttNamesB,symmetricDiff) return (FALSE,message,detailA,detailB) # compare attribute values for varAttName in varAttNamesA: # assuming objects we can compare varAttValueA = eval(r"fpA['"+varName+r"']."+varAttName) varAttValueB = eval(r"fpB['"+varName+r"']."+varAttName) if varAttValueA != varAttValueB: message += ': '+varAttName return (FALSE,message,varAttValueA,varAttValueB) # compare data varDataShapeA = MV.shape(fpA[varName]) varDataShapeB = MV.shape(fpB[varName]) if varDataShapeA != varDataShapeB: message += ': data array shape' return (FALSE,message,str(varDataShapeA),str(varDataShapeB)) maxDelta = MV.maximum(abs(fpA[varName][:] - fpB[varName][:])) if maxDelta > tol: message += ': delta: '+str(maxDelta)+' > '+str(tol) return (FALSE,message,'','') # close files fpA.close() fpB.close() return (TRUE,'','','')
def lonlatobs_to_lonlatmodel(WORKDIR,DIR_REGRID,lon_obs,lat_obs,lon_model,lat_model,sst_nar): #****************************************************************************** # Interpolation horizontale des observations sur la grille du modele # lon_obs,lat_obs => lon_model,lat_model # # Utilisation du module SCRIP #****************************************************************************** import os import cdms, regrid os.chdir(DIR_REGRID) # on se place dans le repertoire de travail scrip_exec='/export/home/logiciels/CDAT/installation/SCRIP/scrip' scrip_in=""" &remap_inputs num_maps = 2 grid1_file = '%s' grid2_file = '%s' interp_file1 = '%s' interp_file2 = '%s' map1_name = '%s Conservative Mapping' map2_name = '%s Conservative Mapping' map_method = 'conservative' normalize_opt = 'frac' output_opt = 'scrip' restrict_type = 'latitude' num_srch_bins = 90 luse_grid1_area = .false. luse_grid2_area = .false. / """ # ------------------------------------------------------------------------------------------ def toScripFormat(slab,file): """ Dumps information about a slab into a file that can be used by the SCRIP regridder Usage toSCRIP(slab,file) where: file: is the output file to which information in SCRIP style will be dumped slab: is the slab from which informations are gathered """ ## Get definitions of grid g=slab.getGrid() print 'lat',g.getMesh()[0,0] print 'lon',g.getMesh()[0,1] m=slab.mask() if m is not None: print m.shape g.setMask(m) print g.shape if g is None: raise 'Error, no grid defined' f=cdms.Cdunif.CdunifFile(file,'w') g.writeScrip(f,'test_scrip') f.close() return # ------------------------------------------------------------------------------------------- #------------------------------------------------------- print '' print '---------- INTERPOLATION HORIZONTALE ----------' print '' #------------------------------------------------------- # Open the SCRIP remapping file and data file direc = '' fremap = cdms.open('rmp_NAR_to_MARS_conserv.nc') # Input data array: sst_nar infile=open(sys.argv[1]) for l in infile.xreadlines(): ## Loop thru input file sp=l.split() f=cdms.open(sp[0]) # Obs file var=sp[1] acc=sp[2] # Accronym for obs finobs=acc+'_obs_SCRIP.nc' s=f(var) while s.rank()>2: ## Gets only lat/lon s=s[0] f.close() ## Creates the obs SCRIP File toScripFormat(s,finobs) qry = 'select accro_orig, version_orig from modelversions,models where models.id=modelversions.model' res=SQL.dbquery(qry) for m in res: ## Prepares strings rmp1='rmp_NAR_to_MARS_conserv.nc' rmp2='rmp_MARS_to_NAR_conserv.nc' tit1='NAR to MARS' tit2='MARS to NAR' fin='remap_grid_NAR.nc' finobs='remap_grid_MARS.nc' scrip_str=scrip_in % ( fin , finobs, rmp1, rmp2, tit1, tit2 ) dbpth=os.path.join(DB.root,'grids',mod_accro) rmp1db=os.path.join(dbpth,rmp1) test = not os.path.exists(rmp1db) test = True if test: print 'Creating remap files between %s and %s' % (mod_accro,acc) ## Reads in grid file fnm=os.path.join(dbpth,mod_accro+'_grid.nc') f=cdms.open(fnm) sm=f('mask')[0] sm=MV.masked_equal(sm,0) toScripFormat(sm,fin) fscr=open('scrip_in','w') print >> fscr,scrip_str fscr.close() ln=os.popen(scrip_exec).readlines() print 'Moving remap files' ln=os.popen('mv %s %s' % (rmp1,rmp1db)).readlines() print ln os.popen('mv %s %s' % (rmp2,os.path.join(dbpth,rmp2))).readlines() os.remove('scrip_in') os.remove(fin) os.remove(finobs) print 'Done' ## Bellow lines to remap using remap files print 'Remapping using : ',rmp1 frmp1=cdms.open(rmp1) remapper=regrid.readRegridder(frmp1) sst_nar_int2D=remapper(s) ## fout=cdms.open(acc+'_remapped_to_1x1.nc','w') ## fout.write(s1) ## fout.close() # Read the SCRIP regridder #regridf = regrid.readRegridder(fremap) # Mettre dans une boucle sur les differents pas de temps # Regrid the variable #sst_nar_int2D = regridf(sst_nar) return sst_nar_int2D
import sys #sys.path.insert(0,'../src/build/lib.linux-i686-2.4') import cdms import spanlib import vcs import MV import Numeric import cdutil import genutil cdms.axis.latitude_aliases.append('Y') cdms.axis.longitude_aliases.append('X') cdms.axis.time_aliases.append('T') f=cdms.open('../example/data2.cdf') s=f('ssta',time=slice(0,120)) print s.shape SP=spanlib.SpAn(s) eof,pc,ev = SP.pca() print 'Done PCA, doing phases w/o mssa' print 'Reconstructing no phases' out = SP.reconstruct() ## phases = spanlib.phases(ffrec) x=vcs.init()
# Needed modules print 'Importing needed modules...' # - base import cdms,MV,vcs,sys,os # - current version of spanlib is prioritary if os.path.exists('../src/build/tmp_lib'):sys.path.insert(0,'../src/build/tmp_lib') import spanlib # We tell cdms that we have longitude, latitude and time cdms.axis.latitude_aliases.append('Y') cdms.axis.longitude_aliases.append('X') cdms.axis.time_aliases.append('T') # Simply open the netcdf file print "Open file" f=cdms.open('data2.cdf') # Get our two datasets print "Read two different regions" s2=f('ssta',latitude=(-10,10),longitude=(110,180)) s1=f('ssta',latitude=(-15,15),longitude=(210,250)) s1[:,0:10,0:4]=MV.masked s2[:,0:5,0:6]=MV.masked print MV.average(s1.flat),MV.average(s2.flat) # Stack the two dataset to have only one dataset print "Stacking data" res = spanlib.stackData(s1,s2) # Create the analysis object print "Creating SpAn object"