def doTCHP(thetaFile, thetaVar, rhoFile, rhoVar, nodata, thresholdTemp, outFile, outFormat, options): print '# open file' fhTheta = cdms2.open(thetaFile) if fhTheta is None: exitMessage("Could not open file {0}. Exit 2.".format(thetaFile), 2) fhRho = cdms2.open(rhoFile) if fhRho is None: exitMessage("Could not open file {0}; Exit(2).".format(rhoFile), 2) thetao = fhTheta.variables[thetaVar][:] # [time, levels, lat, lon] rho = fhRho.variables[rhoVar][:] levelsTmp = fhTheta.variables['lev_bnds'][:] levels = numpy.ravel(0.5*(levelsTmp[:,0] + levelsTmp[:,1] )) print '# mapHeat : time, lat, lon' mapHeat = numpy.zeros( (thetao.shape[0], thetao.shape[2], thetao.shape[3]) ) - 1 print '# Compressing loops...' timelatlon=[] for itime in range(thetao.shape[0]): for ilat in range(thetao.shape[2]): for ilon in range(thetao.shape[3]): if thetao[itime, 0, ilat, ilon] < nodata: timelatlon.append((itime, ilat, ilon)) # loop over time, lat and lon print "parsing the maps" counter=0 for ill in timelatlon: profileTheta = thetao[ ill[0], : ,ill[1], ill[2] ].ravel() profileRho = rho[ ill[0], : ,ill[1], ill[2] ].ravel() heat = 0 heat = computeHeatPotential( profileTheta, profileRho, levels, thresholdTemp, nodata ) mapHeat[ ill[0], ill[1], ill[2] ] = heat #gdal.TermProgress_nocb( counter/float(len(timelatlon)) ) counter = counter+1 #gdal.TermProgress_nocb(1) # save result # outDrv = gdal.GetDriverByName(outformat) # outDS = outDrv.Create(outFile, mapHeat.shape[2], mapHeat.shape[1], mapHeat.shape[0], GDT_Float32, options) # outDS.SetProjection(latlon()) output=cdms2.open(outfile, 'w') output.write(mapHeat) output.close() fhTheta.close() fhRho.close() # for itime in range(thetao.shape[0]): # print '.', # data = numpy.ravel(mapHeat[itime, :, :]) # outDS.GetRasterBand(itime+1).WriteArray( numpy.flipud( data.reshape((mapHeat.shape[1], mapHeat.shape[2])) ) ) # gdal.TermProgress_nocb( itime/float(thetao.shape[0]) ) # gdal.TermProgress_nocb(1) print outDS = None
def setUp(self): self.rootPe = 0 self.pe = MPI.COMM_WORLD.Get_rank() self.clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, ...] # self.tas = cdms2.open(sys.prefix + \ # 'tas_ccsr-95a_1979.01-1979.12.nc')('tas')[0, 0,...] self.tas = cdms2.open(sys.prefix + \ '/sample_data/tas_ecm_1979.nc')('tas')[0, ...] if PLOT: lllat = self.clt.getLatitude()[:].min() urlat = self.clt.getLatitude()[:].max() lllon = self.clt.getLongitude()[:].min() urlon = self.clt.getLongitude()[:].max() self.cmap = bm(llcrnrlat = lllat, llcrnrlon = lllon, urcrnrlat = urlat, urcrnrlon = urlon, resolution = 'i', projection = 'cyl') lllat = self.tas.getLatitude()[:].min() urlat = self.tas.getLatitude()[:].max() lllon = self.tas.getLongitude()[:].min() urlon = self.tas.getLongitude()[:].max() self.tmap = bm(llcrnrlat = lllat, llcrnrlon = lllon, urcrnrlat = urlat, urcrnrlon = urlon, resolution = 'i', projection = 'cyl')
def regrid_5by5(rawOrAnomaly='Anomaly', filteredOrNot='Unfiltered'): """ Written By : Arulalan.T Date : 22.07.2013 """ inpath = os.path.join(processfilesPath, rawOrAnomaly, filteredOrNot) for subName in os.listdir(inpath): anopath = os.path.join(inpath, subName) for anofile in os.listdir(anopath): anofilename = anofile.split('.')[0] varName = anofilename.split('_')[0] if not anofilename.endswith('_5x5'): anofilename += '_5x5' # end of if not anofilename.endswith('_5x5'): anoFilePath = os.path.join(anopath, anofile) outfile = anofilename + '.nc' outpath = os.path.join(anopath, outfile) if os.path.isfile(outpath): print "5x5 regridded file already exists for", outpath continue # end of if os.path.isfile(outpath): grid5x5file = os.path.join(__curDir__, _5x5_ncfilename) varName5x5 = _5x5_ncfilename.split('.')[0] f = cdms2.open(grid5x5file) grid5x5 = f[varName5x5].getGrid() inf = cdms2.open(anoFilePath) grid_infile = inf[varName].getGrid() # Regridding the anomaly data # Creating the horizontal lat,lon regrid # Note that 'grid_infile' is the source and 'grid5x5' is the target regridfunc = Horizontal(grid_infile, grid5x5) anomalytime = inf[varName].getTime().asComponentTime() print "The out path is ", outpath loopCount = len(anomalytime) preview = 0 for day in anomalytime: if __Debug__: print "The Date : ", day data_5x5_regridded = regridfunc(inf(varName, time=day)) outf = cdms2.open(outpath, 'a') outf.write(data_5x5_regridded) outf.close() # make memory free del data_5x5_regridded if __showStatusBar: preview = statusbar(anomalytime.index(day), total=loopCount, title='Regridding', prev=preview) # end of if __showStatusBar: # end of for day in anomalytime: print print "The 5x5 regrid for the variable '%s' is stored %s" % (varName, outpath) inf.close() f.close()
def do_avg(infile, inpath, variable, nodata, outfile): # for netcdf3: set flags to 0 cdms2.setNetcdfShuffleFlag(1) cdms2.setNetcdfDeflateFlag(1) cdms2.setNetcdfDeflateLevelFlag(3) # note that this version will erase data whereever a nodata is found in the series avg=None nodatamask = None for ifile in infile: fname = os.path.join(inpath, ifile) if not os.path.exists(fname): messageOnExit('file {0} not found on path {1}. Exit(100).'.format(ifile, path), 100) thisfile = cdms2.open(fname, 'r') if avg is None: avg = numpy.array(thisfile[variable][:]) nodatamask = avg >= nodata else: avg = avg + numpy.array(thisfile[variable][:]) thisfile.close() avg = avg/len(infile) if nodatamask.any(): avg[nodatamask] = nodata if os.path.exists(outfile): os.remove(outfile) outfh = cdms2.open(outfile, 'w') outvar=cdms2.createVariable(avg, typecode='f', id=variable, fill_value=nodata ) outfh.write(outvar) outfh.close()
def test4_cdmsRegrid(self): srcF = cdms2.open(sys.prefix + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = srcF('so')[0, 0, ...] srcGridMask = numpy.array((so == so.missing_value) , numpy.int32) clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt') # create regrid object r = cdms2.CdmsRegrid(so.getGrid(), clt.getGrid(), dtype=so.dtype, regridMethod='conserve', regridTool='esmf', srcGridMask=srcGridMask, srcGridAreas=None, dstGridMask=None, dstGridAreas=None) dstData = r(so) # checks if self.pe == 0: dstDataMask = (dstData == so.missing_value) print 'number of masked values = ', dstDataMask.sum() self.assertTrue(str(type(dstData)), str(type(clt))) dstData.mask = (dstData == so.missing_value) dstDataMin = dstData.min() dstDataMax = dstData.max() zeroValCnt = (dstData == 0).sum() print 'Number of zero valued cells', zeroValCnt print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax) self.assertLess(dstDataMax, so.max())
def setUp(self): filename = "so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc" h=cdms2.open('./' + filename) self.hso = h('so')[0, 0, ...] self.hGrid = [self.hso.getLatitude(), self.hso.getLongitude()] self.grid2D = self.hso.getGrid() filename = "clt.nc" f=cdms2.open(filename) self.fclt = f('clt') self.fGrid = [self.fclt.getLatitude(), self.fclt.getLongitude()] g2D = self.fclt.getGrid().toCurveGrid() self.grid2D = [g2D.getLatitude()[:], g2D.getLongitude()[:]] filename = "era40_tas_sample.nc" g=cdms2.open(filename) self.gtas = g('tas') self.gGrid = [self.gtas.getLatitude(), self.gtas.getLongitude()] g2D = self.gtas.getGrid().toCurveGrid() self.grid2D = [g2D.getLatitude()[:], g2D.getLongitude()[:]] self.gcltVal010101 = 74.0087890625 self.ftasVal1010 = 283.39410400390625 self.soVal1010 = 33.88642120361328 self.MissingValue = 1.e20 self.eps = 1e-4
def XXtest5_regrid(self): srcF = cdms2.open(sys.prefix + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = srcF('so')[0, 0, ...] clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt') dstData = so.regrid(clt.getGrid(), regridTool = 'esmf', regridMethod='conserve') if self.pe == 0: dstDataMask = (dstData == so.missing_value) dstDataFltd = dstData * (1 - dstDataMask) zeroValCnt = (dstData == 0).sum() if so.missing_value > 0: dstDataMin = dstData.min() dstDataMax = dstDataFltd.max() else: dstDataMin = dstDataFltd.min() dstDataMax = dstData.max() zeroValCnt = (dstData == 0).sum() print 'Number of zero valued cells', zeroValCnt print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax) self.assertLess(dstDataMax, so.max()) if False: pylab.figure(1) pylab.pcolor(so, vmin=20, vmax=40) pylab.colorbar() pylab.title('so') pylab.figure(2) pylab.pcolor(dstData, vmin=20, vmax=40) pylab.colorbar() pylab.title('dstData')
def test_pack(c): for pack in [False, True]: # print "Testing pack:",pack f = cdms2.open("test_pack.nc", "w") f.write(c, id="packed", pack=pack) f.close() f = cdms2.open("test_pack.nc") d = f("packed") f.close() assert c.dtype == d.dtype if pack in [True, numpy.int16]: atol = rtol = 1.0 / pow(2, 11) # elif pack in [numpy.int,numpy.int64]: # atol=rtol = 1./pow(2,63) # elif pack == numpy.int32: # atol=rtol = 1./pow(2,31) # elif pack == numpy.int8: # atol=rtol = 1./pow(2,7) elif pack is False: atol = rtol = 0.0 # print "M:",pack,c.max(),d.max() assert numpy.ma.allclose(numpy.ma.maximum(c), numpy.ma.maximum(d), True, rtol, atol) # print "m:",pack,c.min(),d.min() assert numpy.ma.allclose(numpy.ma.minimum(c), numpy.ma.minimum(d), True, rtol, atol) # print "D:",pack, (c-d).max(),(c-d).min(),((c-d)/c).max(),rtol assert numpy.ma.allclose(c, d, True, rtol, atol) os.remove("test_pack.nc")
def download_file(filename, varlist, proxy_cert): outfilename="/tmp/" + str(time.time()) myvarlist=varlist.split(',') if len(myvarlist)>0 and myvarlist[0] != '': try: f=cdms2.open(filename) fout=cdms2.open(outfilename,'w') for var in myvarlist: myvar=str(var) fout.write(f(myvar,time=slice(0,1))) fout.close() f.close() return outfilename except Exception as e: print e return None else: try: wget_url="http://pcmdi9.llnl.gov/esg-search/wget?url="+filename print wget_url with open(outfilename,'w') as outfile: flock(outfile,LOCK_EX) cmd="wget --certificate %s -t 2 -T 10 --private-key %s -O %s %s"%(proxy_cert,proxy_cert,outfilename,wget_url) output=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True) time.sleep(.1) flock(outfile,LOCK_UN) outfile.close() return outfilename except Exception as e: return None
def closeness( self, varname, filename, rtol, atol ): #pdb.set_trace() testfname = os.path.join( self.outpath, filename ) #print '>>>>>>>>>>>>>>>>>>> ', testfname baselinefname = os.path.join( self.baselinepath, filename ) #print '>>>>>>>>>>>>>>>>>>> ', baselinefname f = cdms2.open( testfname ) g = cdms2.open( baselinefname ) try: fvar = f(varname) except: print varname, ' is not in file ', testname sys.exit(1) try: gvar = g(varname) except: print varname, ' is not in file ', baselinefname sys.exit(1) #print '>>>>>>>>>>>>>>>>>>> fvar', fvar.shape #print '>>>>>>>>>>>>>>>>>>> gvar', gvar.shape close = numpy.ma.allclose( fvar, gvar, rtol=rtol, atol=atol ) if close: print "fvar and gvar are close for", varname else: print "fvar and gvar differ for", varname print "max difference", (fvar-gvar).max() print "min difference", (fvar-gvar).min() f.close() g.close() return close
def setUp(self): """ Set up the grids to pass to mvGenericRegrid """ self.doPlots = False self.clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, ...] # Convert to curvilinear cds, cds_forBounds = [], [] cds.append(self.clt.getLatitude()) cds.append(self.clt.getLongitude()) cds_forBounds.append(self.clt.getLatitude()) cds_forBounds.append(self.clt.getLongitude()) self.cltGrid, nDims = regrid2.gsRegrid.makeCurvilinear(cds) self.cltInterp = numpy.array(self.clt) * 0.0 + self.clt.missing_value self.cltInterpInterp = numpy.array(self.clt) * 0.0 + self.clt.missing_value # Salinity check f = cdms2.open(sys.prefix + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') self.so = f('so') soGrid = [] soGrid.append(self.so.getLatitude()) soGrid.append(self.so.getLongitude()) self.soGrid = soGrid self.soInterp = numpy.array(self.clt) * 0.0 + self.so.missing_value self.tol = 1e2 self.comm = MPI.COMM_WORLD self.rank = self.comm.Get_rank() self.size = self.comm.Get_size()
def initialize( self, args, **cfg_args ): self.configure( **cfg_args ) ( grid_file, data_file, interface, varname, grid_coords, var_proc_op ) = args self.gf = cdms2.open( grid_file ) if grid_file else None self.df = cdms2.open( data_file ) self.var = self.getProcessedVariable( varname, var_proc_op ) self.extractMetadata() self.grid = self.var.getGrid() self.lev = self.getLevel(self.var) lon, lat = self.getLatLon( grid_coords ) if ( id(lon) <> id(None) ) and ( id(lat) <> id(None) ): self.time = self.var.getTime() z_scale = 0.5 self.missing_value = self.var.attributes.get( 'missing_value', None ) if self.lev == None: domain = self.var.getDomain() for axis in domain: if PlotType.isLevelAxis( axis[0].id.lower() ): self.lev = axis[0] break self.computePoints() self.setPointHeights( height_var=grid_coords[3], z_scale=z_scale ) np_var_data_block = self.getDataBlock(self.var).flatten() if self.missing_value: var_data = numpy.ma.masked_equal( np_var_data_block, self.missing_value, False ) else: var_data = np_var_data_block self.point_data_arrays[ self.defvar ] = var_data self.vrange = ( var_data.min(), var_data.max() ) self.var_data_cache[ self.iTimeStep ] = var_data self.metadata['vbounds'] = self.vrange
def coarsenVars( filen, newfilen ): """Reads a NetCDF file, filen, and writes a new file, newfilen, containing all its variables, but regridded in its horizontal axeseb to a 12x14 (15 degree) lat-lon grid. """ latbnds = numpy.linspace(-90,90,nlat) lat = 0.5*( latbnds[1:]+latbnds[:-1] ) lataxis=cdms2.createAxis( lat, id='lat' ) lonbnds = numpy.linspace(0,360,nlon) lon = 0.5*( lonbnds[1:]+lonbnds[:-1] ) lonaxis=cdms2.createAxis( lon, id='lon' ) newgrid=cdms2.createRectGrid(lataxis, lonaxis) f = cdms2.open(filen) nf = cdms2.open(newfilen,'w') for varn in f.variables.keys(): print "working on",varn var = f(varn) if var.getGrid() is None: print varn,"has no grid!" continue newvar = var.regrid(newgrid) # works even if var has levels newvar.id = var.id # before, id was var.id+'_CdmaRegrid' nf.write(newvar) for attr in f.attributes.keys(): setattr(nf,attr,getattr(f,attr)) if hasattr(nf,'history'): nf.history = nf.history + '; some variables deleted' f.close() nf.close()
def getData(self, variable=None): ''' Aggreate all data from the file list in the order given. ''' filename = self.flist[0].strip() f = cdms2.open( filename, 'r' ) if( variable != None ): self.vartoread = variable else: self.vartoread = self.variable.id data = f(self.vartoread)[:] # --------------------------- # Concatenate following files # --------------------------- for filename in self.flist[ 1: ]: print "reading %s" % filename.strip() f = cdms2.open( filename.strip(), 'r' ) data2 = f(self.vartoread)[:] data = numpy.concatenate((data,data2), axis=0) f.close() data=ma.array(data=data, \ fill_value=self.missing_value, \ copy=0, \ dtype='float32' ) data=ma.masked_equal(data, self.missing_value, copy=0) return data
def test2_varRegrid(self): print print 'test2_varRegrid' srcF = cdms2.open(sys.prefix + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = srcF('so')[0, 0, ...] clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt') diag = {'srcAreas': None, 'dstAreas': None, 'srcAreaFractions': None, 'dstAreaFractions': None} soInterp = so.regrid(clt.getGrid(), regridTool = 'esmf', regridMethod='conserve', diag = diag) if self.pe == 0: totSrcArea = diag['srcAreas'].sum() totDstArea = diag['dstAreas'].sum() totSrcFrac = diag['srcAreaFractions'].sum() self.assertEqual(numpy.isnan(totSrcFrac).sum(), 0) self.assertLess(abs(totSrcArea - 4*pi)/(4*pi), 0.02) self.assertLess(abs(totDstArea - 4*pi)/(4*pi), 0.01) soMass = (so*diag['srcAreas']).sum() inMass = (soInterp*diag['dstAreas']).sum() print soMass, inMass diff = abs(soMass - inMass)/soMass self.assertLess(diff, 7.e-7) if False: pylab.subplot(1, 2, 1) pylab.pcolor(so, vmin = 20, vmax = 40) pylab.colorbar() pylab.title('so') pylab.subplot(1, 2, 2) pylab.pcolor(soInterp, vmin = 20, vmax = 40) pylab.colorbar() pylab.title('soInterp')
def test_2d_esmf_conserv(self): print 'running test_2d_esmf_conserv...' f = cdms2.open(sys.prefix + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = f('so')[0, 0, :, :] clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, :, :] tic = time.time() soInterp = so.regrid(clt.getGrid(), regridTool='ESMF', regridMethod='Conservative') soInterpInterp = soInterp.regrid(so.getGrid(), regridTool='ESMF', regridMethod='Conservative') toc = time.time() print 'time to interpolate (ESMF conservative) forward/backward: ', toc - tic ntot = reduce(operator.mul, so.shape) avgdiff = numpy.sum(so - soInterpInterp) / float(ntot) print 'avgdiff = ', avgdiff if PLOT: pylab.figure(2) pylab.subplot(2, 2, 1) pylab.pcolor(so, vmin=20.0, vmax=40.0) pylab.colorbar() pylab.title('ESMF conserve regrid: so') pylab.subplot(2, 2, 2) pylab.pcolor(soInterp, vmin=20.0, vmax=40.0) pylab.colorbar() pylab.title('ESMF conserve regrid: soInterp') pylab.subplot(2, 2, 3) pylab.pcolor(soInterpInterp, vmin=20.0, vmax=40.0) pylab.colorbar() pylab.title('ESMF conserve regrid: soInterpInterp') pylab.subplot(2, 2, 4) pylab.pcolor(so - soInterpInterp, vmin=-0.5, vmax=0.5) pylab.colorbar() pylab.title('ESMF conserve regrid: error')
def demoScene(self): from sys import executable from os.path import split,join import cdms2 sample_dir = join(split(executable)[0],"..","sample_data") mean = cdms2.open(join(sample_dir,"mean_1900_1_1900_12.nc"))['tas'] std = cdms2.open(join(sample_dir,"std_1900_1_1900_12.nc"))['tas'] cloudiness = cdms2.open(join(sample_dir,"cl_A1.20C3M_1.CCSM.atmm.1900-01_cat_1909-12.nc"))['cl'] mean = mean(longitude=(-180,180)) std = std(longitude=(-180,180)) mean.getTime().toRelativeTime(cloudiness.getTime().units) std.getTime().toRelativeTime(cloudiness.getTime().units) t = mean.getTime() t = [t[0],t[-1]] cloudiness = cloudiness(longitude=(-180,180),time=t) scene = VisusMovieScene.MovieScene(mean, std, cloudiness) pyvisus.gVCDATScenes += [scene]
def test4_cdmsRegrid(self): srcF = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = srcF('so')[0, 0, ...] clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt') # create regrid object r = cdms2.CdmsRegrid(so.getGrid(), clt.getGrid(), so.dtype, regridMethod='linear', regridTool='libcf', srcGridMask=(so == so.missing_value), srcGridAreas=None, dstGridMask=None, dstGridAreas=None, mkCyclic=False, handleCut=False) dstData = r(so) # checks dstDataMask = (dstData == so.missing_value) dstDataFltd = dstData * (1 - dstDataMask) if so.missing_value > 0: dstDataMin = dstData.min() dstDataMax = dstDataFltd.max() else: dstDataMin = dstDataFltd.min() dstDataMax = dstData.max() print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax) self.assertGreater(dstDataMin, so.min()) self.assertLess(dstDataMax, so.max())
def Xtest1_gsRegrid(self): srcF = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = srcF('so')[0, 0, ...] clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt') srcGrd = [so.getGrid().getLatitude(), so.getGrid().getLongitude()] dstGrd = [clt.getGrid().getLatitude(), clt.getGrid().getLongitude()] # create regrid object r = regrid2.gsRegrid.Regrid(srcGrd, dstGrd, src_bounds=None, mkCyclic=False, handleCut=False) # set mask r.setMask( (so == so.missing_value) ) # compute interpolation weights r.computeWeights(nitermax=20, tolpos=0.01) # make sure most of the weights have ben found print 'ratio of valid over number of nodes: %f' % (r.getNumValid()/float(r.getNumDstPoints())) self.assertGreater(r.getNumValid(), 0) # create dst data container dstShp = r.getDstGrid()[0].shape print 'dst data shape: ', dstShp dstData = numpy.ones(dstShp, so.dtype) * so.missing_value # interpolate r.apply(so, dstData) # checks dstDataMask = (dstData == so.missing_value) dstDataFltd = dstData * (1 - dstDataMask) if so.missing_value > 0: dstDataMin = dstData.min() dstDataMax = dstDataFltd.max() else: dstDataMin = dstDataFltd.min() dstDataMax = dstData.max() print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax) self.assertGreater(dstDataMin, so.min()) self.assertLess(dstDataMax, so.max())
def Xtest3_genericRegrid(self): srcF = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') so = srcF('so')[0, 0, ...] clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt') srcGrd = [so.getGrid().getLatitude(), so.getGrid().getLongitude()] dstGrd = [clt.getGrid().getLatitude(), clt.getGrid().getLongitude()] # create regrid object r = regrid2.mvGenericRegrid.GenericRegrid(srcGrd, dstGrd, so.dtype, regridMethod='linear', regridTool='libcf', srcGridMask=(so == so.missing_value), srcBounds=None, srcGridAreas=None, dstGridMask=None, dstBounds=None, dstGridAreas=None, mkCyclic=False, handleCut=False) # compute weights r.computeWeights(nitermax=20, tolpos=0.01) # create dst data container dstShp = r.getDstGrid()[0].shape dstData = numpy.ones(dstShp, so.dtype) * so.missing_value # interpolate r.apply(so, dstData) # checks dstDataMask = (dstData == so.missing_value) dstDataFltd = dstData * (1 - dstDataMask) if so.missing_value > 0: dstDataMin = dstData.min() dstDataMax = dstDataFltd.max() else: dstDataMin = dstDataFltd.min() dstDataMax = dstData.max() print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax) self.assertGreater(dstDataMin, so.min()) self.assertLess(dstDataMax, so.max())
def setUp(self): self.so = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')('so') self.tasGood = cdms2.open(cdat_info.get_sampledata_path() + \ '/tas_Amon_HadGEM2-A_amip_r1i2p1_197809-200811_2timesteps.nc')('tas') self.tasBad = cdms2.open(cdat_info.get_sampledata_path() + \ '/tas_Amon_ACCESS1-0_historical_r1i1p1_185001-189912_2timesteps.nc')('tas')
def test_test2(self): """ 2D ESMP """ u = cdms2.open(sys.prefix + "/sample_data/clt.nc")("u")[0, 0, ...] clt = cdms2.open(sys.prefix + "/sample_data/clt.nc")("clt")[0, ...] ctlOnUGrid = clt.regrid(u.getGrid(), regridTool="ESMP") # print 'ctlOnUGrid.getGrid() = ', type(ctlOnUGrid.getGrid()) self.assertRegexpMatches(str(type(ctlOnUGrid.getGrid())), "cdms2.grid.TransientRectGrid")
def setUp(self): fso = cdms2.open(sys.prefix + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') self.soLevel = fso('so')[0,0:5,...] self.soTime = fso('so')[:,0,...] self.so = fso('so')[:,0:5,...] fclt = cdms2.open(sys.prefix + 'sample_data/clt.nc') self.clt = fclt('clt')
def setUp(self): fso = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') self.soLevel = fso('so')[0,0:5,...] self.soTime = fso('so')[:,0,...] self.so = fso('so')[:,0:5,...] fclt = cdms2.open(cdat_info.get_sampledata_path() + "/clt.nc") self.clt = fclt('clt')
def initialize( self, args, **cfg_args ): self.configure( **cfg_args ) ( grid_file, data_file, interface, grd_varnames, grd_coords, var_proc_op, ROI, subSpace ) = args self.interface = interface self.roi = ROI self.gf = cdms2.open( grid_file ) if grid_file else None self.df = cdms2.open( data_file ) self.grid_varname = grd_varnames[0] if ( grd_varnames <> None ) else self.df.variables[0] self.grid_coords = grd_coords self.initPoints( var_proc_op )
def setUp(self): f = cdms2.open('so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') self.accessModel = f('so') h = cdms2.open('so_Omon_HadGEM2-CC_historical_r1i1p1_185912-186911_2timesteps.nc') self.hadGEM2Model = h('so') self.clt = cdms2.open('clt.nc')('clt')[0, :, :] self.eps = 1e-5
def test_test2(self): """ 2D ESMP """ u = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('u')[0, 0,...] clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')[0, ...] ctlOnUGrid = clt.regrid( u.getGrid(), regridTool = "ESMP" ) #print 'ctlOnUGrid.getGrid() = ', type(ctlOnUGrid.getGrid()) self.assertRegexpMatches(str(type(ctlOnUGrid.getGrid())), "cdms2.grid.TransientRectGrid")
def setUp(self): """ Set up the grids to pass to mvGenericRegrid """ f = cdms2.open(cdat_info.get_prefix() + \ '/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') self.so = f('so') g = cdms2.open(cdat_info.get_prefix() + '/sample_data/clt.nc') self.clt = g('clt')
def loadFileFromURL(self,url): ## let's figure out between dap or local if url[:7].lower()=="http://": f=cdms2.open(str(url)) elif url[:7]=="file://": f=cdms2.open(str(url[6:])) else: # can't figure it out skipping f=None return f
def setUp(self): access = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc') self.access = access('so')[0, 0, ...] self.tasGood = cdms2.open(cdat_info.get_sampledata_path() + \ '/tas_Amon_HadGEM2-A_amip_r1i2p1_197809-200811_2timesteps.nc')('tas') giss = cdms2.open(cdat_info.get_sampledata_path() + \ '/so_Omon_GISS-E2-R_historicalNat_r5i1p1_185001-187512_2timesteps.nc') self.giss = giss('so')[0,0,...]
# Adapted for numpy/ma/cdms2 by convertcdms.py import vcs_legacy, cdms2 as cdms, os, sys, support bg = support.bg tp = vcs_legacy.init() tp.portrait() fnm = os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'sample_data', 'clt.nc') f = cdms.open(fnm) s = f('clt') tp.plot(s, bg=bg) support.check_plot(tp) tit = tp.createtext('tit') tit.x = [0.5] tit.y = [.9] tit.halign = 'center' tit.height = 20 tit.color = 241 n = 6 i = 0 tit.string = 'Line %d - Decorrelation Time of Volume Mean Temperature Change' % i tp.plot(tit, bg=bg) support.check_plot(tp) tp.mode = 1 for i in range(1, n): tit.y = [1. - (i / float(n))] support.check_plot(tp)
import cdms2, sys, numpy, cdat_info f = cdms2.open(cdat_info.get_sampledata_path() + "/clt.nc") s = f("clt") jsn = s.dumps() s2 = cdms2.createVariable(jsn, fromJSON=True) assert (numpy.allclose(s2, s))
def compute(params): fileName = params.fileName startyear = params.args.firstyear finalyear = params.args.lastyear month = params.args.month monthname = params.monthname varbname = params.varname template = populateStringConstructor(args.filename_template, args) template.variable = varbname reverted = template.reverse(os.path.basename(fileName)) dataname = reverted["model"] if dataname not in args.skip: try: print('Data source:', dataname) print('Opening %s ...' % fileName) f = cdms2.open(fileName) iYear = 0 dmean = None for year in range(startyear, finalyear + 1): print('Year %s:' % year) startTime = cdtime.comptime(year, month) # Last possible second to get all tpoints finishtime = startTime.add(1, cdtime.Month).add( -1, cdtime.Minute) print('Reading %s from %s for time interval %s to %s ...' % (varbname, fileName, startTime, finishtime)) # Transient variable stores data for current year's month. tvarb = f(varbname, time=(startTime, finishtime, "ccn")) # *HARD-CODES conversion from kg/m2/sec to mm/day. tvarb *= 86400 # The following tasks need to be done only once, extracting # metadata from first-year file: tc = tvarb.getTime().asComponentTime() current = tc[0] while current.month == month: end = cdtime.comptime(current.year, current.month, current.day).add(1, cdtime.Day) sub = tvarb(time=(current, end, "con")) # Assumes first dimension of input ("axis#0") is time tmp = numpy.ma.average(sub, axis=0) sh = list(tmp.shape) sh.insert(0, 1) if dmean is None: dmean = tmp.reshape(sh) else: dmean = numpy.ma.concatenate((dmean, tmp.reshape(sh)), axis=0) current = end iYear += 1 f.close() stdvalues = cdms2.MV2.array(genutil.statistics.std(dmean)) stdvalues.setAxis(0, tvarb.getLatitude()) stdvalues.setAxis(1, tvarb.getLongitude()) stdvalues.id = 'dailySD' # Standard deviation has same units as mean. stdvalues.units = "mm/d" stdoutfile = ('%s_%s_%s_%s-%s_std_of_dailymeans.nc') % ( varbname, dataname, monthname, str(startyear), str(finalyear)) except Exception as err: print("Failed for model: %s with error: %s" % (dataname, err)) if not os.path.exists(args.results_dir): os.makedirs(args.results_dir) g = cdms2.open(os.path.join(args.results_dir, stdoutfile), 'w') g.write(stdvalues) g.close()
# # ############################################################################ # # # Import: VCS and cdms modules. # # # ############################################################################ import vcs_legacy import cdms2 as cdms import sys import os import support f = cdms.open( os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'sample_data', 'meshfill.nc')) M = f('Mesh') s = f('Data') x = vcs_legacy.init() x.scriptrun( os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'bin', 'ASD.scr')) m = x.getmeshfill('ASD') m.wrap = [0, 360] m.mesh = 'y' #mn,mx=vcs_legacy.minmax(s) levs = vcs_legacy.mkscale(-10, 30) #print levs levs = vcs_legacy.mkevenlevels(levs[0], levs[-1], 256) levs2 = levs[::25]
import os from durolib import globalAttWrite import cdms2 as cdm import numpy as np import MV2 as mv # Set netcdf file criterion - turned on from default 0s cdm.setCompressionWarnings(0) # Suppress warnings cdm.setNetcdfShuffleFlag(0) cdm.setNetcdfDeflateFlag(1) cdm.setNetcdfDeflateLevelFlag(9) cdm.setAutoBounds(1) # JRA25 infile = '/work/durack1/Shared/obs_data/pcmdi_metrics/197901/anl_land25.ctl' f_h = cdm.open(infile) soilwhbl = f_h('soilwhbl')[0, 0, ...] landMask = soilwhbl.mask lat = soilwhbl.getLatitude() lon = soilwhbl.getLongitude() # Convert to percentage landMask = landMask * 100 # Convert to boolean fraction # Rename landMask = cdm.createVariable( landMask, id='sftlf', axes=soilwhbl.getAxisList(), typecode='float32') landMask.original_name = "soilwhbl"
def compute(params): fileName = params.fileName month = params.args.month monthname = params.monthname varbname = params.varname template = populateStringConstructor(args.filename_template, args) template.variable = varbname # Units on output (*may be converted below from the units of input*) outunits = 'mm/d' startime = 1.5 # GMT value for starting time-of-day dataname = params.args.model if dataname is None or dataname.find("*") != -1: # model not passed or passed as * reverted = template.reverse(os.path.basename(fileName)) print("REVERYING", reverted, dataname) dataname = reverted["model"] if dataname not in args.skip: try: print('Data source:', dataname) print('Opening %s ...' % fileName) f = cdms2.open(fileName) # Composite-mean and composite-s.d diurnal cycle for month and year(s): iYear = 0 for year in range(args.firstyear, args.lastyear + 1): print('Year %s:' % year) startTime = cdtime.comptime(year, month) # Last possible second to get all tpoints finishtime = startTime.add(1, cdtime.Month).add( -1, cdtime.Minute) print('Reading %s from %s for time interval %s to %s ...' % (varbname, fileName, startTime, finishtime)) # Transient variable stores data for current year's month. tvarb = f(varbname, time=(startTime, finishtime)) # *HARD-CODES conversion from kg/m2/sec to mm/day. tvarb *= 86400 print('Shape:', tvarb.shape) # The following tasks need to be done only once, extracting # metadata from first-year file: if year == args.firstyear: tc = tvarb.getTime().asComponentTime() print("DATA FROM:", tc[0], "to", tc[-1]) day1 = cdtime.comptime(tc[0].year, tc[0].month) day1 = tc[0] firstday = tvarb(time=(day1, day1.add(1., cdtime.Day), "con")) dimensions = firstday.shape print(' Shape = ', dimensions) # Number of time points in the selected month for one year N = dimensions[0] nlats = dimensions[1] nlons = dimensions[2] deltaH = 24. / N dayspermo = tvarb.shape[0] // N print( ' %d timepoints per day, %d hr intervals between timepoints' % (N, deltaH)) comptime = firstday.getTime() modellons = tvarb.getLongitude() modellats = tvarb.getLatitude() # Longitude values are needed later to compute Local Solar # Times. lons = modellons[:] print(' Creating temporary storage and output fields ...') # Sorts tvarb into separate GMTs for one year tvslice = MV2.zeros((N, dayspermo, nlats, nlons)) # Concatenates tvslice over all years concatenation = MV2.zeros( (N, dayspermo * nYears, nlats, nlons)) LSTs = MV2.zeros((N, nlats, nlons)) for iGMT in range(N): hour = iGMT * deltaH + startime print( ' Computing Local Standard Times for GMT %5.2f ...' % hour) for j in range(nlats): for k in range(nlons): LSTs[iGMT, j, k] = (hour + lons[k] / 15) % 24 for iGMT in range(N): hour = iGMT * deltaH + startime print(' Choosing timepoints with GMT %5.2f ...' % hour) print("days per mo :", dayspermo) # Transient-variable slice: every Nth tpoint gets all of # the current GMT's tpoints for current year: tvslice[iGMT] = tvarb[iGMT::N] concatenation[iGMT, iYear * dayspermo:(iYear + 1) * dayspermo] = tvslice[iGMT] iYear += 1 f.close() # For each GMT, take mean and standard deviation over all years for # the chosen month: avgvalues = MV2.zeros((N, nlats, nlons)) stdvalues = MV2.zeros((N, nlats, nlons)) for iGMT in range(N): hour = iGMT * deltaH + startime print( 'Computing mean and standard deviation over all GMT %5.2f timepoints ...' % hour) # Assumes first dimension of input ("axis#0") is time avgvalues[iGMT] = MV2.average(concatenation[iGMT], axis=0) stdvalues[iGMT] = genutil.statistics.std(concatenation[iGMT]) avgvalues.id = 'diurnalmean' stdvalues.id = 'diurnalstd' LSTs.id = 'LST' avgvalues.units = outunits # Standard deviation has same units as mean (not so for # higher-moment stats). stdvalues.units = outunits LSTs.units = 'hr' LSTs.longname = 'Local Solar Time' avgvalues.setAxis(0, comptime) avgvalues.setAxis(1, modellats) avgvalues.setAxis(2, modellons) stdvalues.setAxis(0, comptime) stdvalues.setAxis(1, modellats) stdvalues.setAxis(2, modellons) LSTs.setAxis(0, comptime) LSTs.setAxis(1, modellats) LSTs.setAxis(2, modellons) avgoutfile = ('%s_%s_%s_%s-%s_diurnal_avg.nc') % ( varbname, dataname, monthname, str( args.firstyear), str(args.lastyear)) stdoutfile = ('%s_%s_%s_%s-%s_diurnal_std.nc') % ( varbname, dataname, monthname, str( args.firstyear), str(args.lastyear)) LSToutfile = ('%s_%s_LocalSolarTimes.nc' % (varbname, dataname)) if not os.path.exists(args.results_dir): os.makedirs(args.results_dir) f = cdms2.open(os.path.join(args.results_dir, avgoutfile), 'w') g = cdms2.open(os.path.join(args.results_dir, stdoutfile), 'w') h = cdms2.open(os.path.join(args.results_dir, LSToutfile), 'w') f.write(avgvalues) g.write(stdvalues) h.write(LSTs) f.close() g.close() h.close() except Exception as err: print("Failed for model %s with erro: %s" % (dataname, err))
# Adapted for numpy/ma/cdms2 by convertcdms.py import cdms2, sys, ZonalMeans cdms2.setAutoBounds("on") cdms2.axis.level_aliases.append('depth') import vcs import vcs.test.support bg = vcs.test.support.bg x = vcs.init() # Open data file f = cdms2.open('test_data.nc') O2 = f("O2", level=slice(0, 5)) print O2.shape, O2.mask print 'Level:', O2.getLevel() f.close() # Open grid file f = cdms2.open('test_grid.nc') # 1st with just bounds blon = f('bounds_lon') blat = f('bounds_lat') #mask = 1 - f('mask') area = f('area') res = ZonalMeans.compute(O2, bounds_lon=blon, bounds_lat=blat, area=area, delta_band=5) print res[1].shape x.plot(res[1][:, 0], bg=bg)
def Xtest_test1(self): """ Test case with domain decomposition """ f = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc', 'r') cltVar = f['clt'] # global sizes nLat, nLon = cltVar.shape[1:] # local rank and number of procs rk = MPI.COMM_WORLD.Get_rank() sz = MPI.COMM_WORLD.Get_size() # compute domain decomposition npLat, npLon = getDomainDecomp(sz, (nLat, nLon)) if rk == 0: print '' print 'lat x lon sizes : %d x %d ' % (nLat, nLon) print 'lat x lon domain decomp: %d x %d ' % (npLat, npLon) # number of points in each local domain nlat, nlon = nLat / npLat, nLon / npLon ipLat = rk // npLon ipLon = rk % npLon # starting/ending indices for local domain iLatBeg, iLatEnd = ipLat * nlat, (ipLat + 1) * nlat iLonBeg, iLonEnd = ipLon * nlon, (ipLon + 1) * nlon # now read local domain data for time 0 clt = cltVar[0, iLatBeg:iLatEnd, iLonBeg:iLonEnd] # make halo available to other procs numGhosts = 1 clt.exposeHalo(ghostWidth=numGhosts) # find the procs to the north, east, south, and west noProc = min(ipLat + 1, npLat - 1) * npLon + ipLon + 0 soProc = max(ipLat - 1, 0) * npLon + ipLon + 0 # warp around eaProc = (ipLat + 0) * npLon + (ipLon + 1) % npLon weProc = (ipLat + 0) * npLon + (ipLon - 1) % npLon # fetch the remote data noCltData = clt.fetchHaloData(noProc, side=(-1, 0)) soCltData = clt.fetchHaloData(soProc, side=(+1, 0)) eaCltData = clt.fetchHaloData(eaProc, side=(0, -1)) weCltData = clt.fetchHaloData(weProc, side=(0, +1)) # slabs in local index space noSlc = clt.getHaloEllipsis(side=(-1, 0)) soSlc = clt.getHaloEllipsis(side=(+1, 0)) eaSlc = clt.getHaloEllipsis(side=(0, -1)) weSlc = clt.getHaloEllipsis(side=(0, +1)) tol = 1.e-10 success = True if True: # north if ipLat < npLat - 1: print 'ipLat < npLat - 1:', ipLat, npLat - 1 igLatBeg = (ipLat + 1) * nlat igLatEnd = igLatBeg + numGhosts igLonBeg = ipLon * nlon igLonEnd = igLonBeg + nlon globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd] if abs(numpy.sum(globalCltData - noCltData)) > tol: success = False # south if ipLat >= 1: print 'ipLat >= 1:', ipLat, npLat - 1 igLatBeg = (ipLat + 0) * nlat - numGhosts igLatEnd = igLatBeg + numGhosts igLonBeg = ipLon * nlon igLonEnd = igLonBeg + nlon globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd] if abs(numpy.sum(globalCltData - soCltData)) > tol: success = False # east igLatBeg = ipLat * nlat igLatEnd = igLatBeg + nlat igLonBeg = ((ipLon + 1) % npLon) * nlon igLonEnd = igLonBeg + numGhosts globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd] if abs(numpy.sum(globalCltData - eaCltData)) > tol: success = False # west igLatBeg = ipLat * nlat igLatEnd = igLatBeg + nlat igLonBeg = ((ipLon - 1) % npLon) * nlon + nlon - numGhosts igLonEnd = igLonBeg + numGhosts globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd] if abs(numpy.sum(globalCltData - weCltData)) > tol: success = False self.assertEqual(success, True) clt.freeHalo()
import cdms2 import os import sys import vcs cdmsfile = cdms2.open(os.path.join(vcs.prefix, "sample_data", "clt.nc")) data = cdmsfile('clt') x = vcs.init() t = x.gettemplate('default') x.plot(data, t, bg=True) # This should force the image to update x.setcolormap('bl_to_drkorang') testFilename = "test_vcs_setcolormap.png" x.png(testFilename)
import os, sys, cdms2, vcs, vcs.testing.regression as regression # Load the clt data: dataFile = cdms2.open(os.path.join(vcs.sample_data, "clt.nc")) clt = dataFile("clt", slice(0, 1), squeeze=1) height, width = clt.shape clt.mask = [[True if i % 2 else False for i in range(width)] for _ in range(height)] # Initialize canvas: canvas = regression.init() # Create and plot quick boxfill with default settings: # Only have to test boxfill because all 2D methods use the same code # for missing values boxfill = canvas.createboxfill() # Change the missing color to a colorname boxfill.missing = "lightgrey" canvas.plot(clt, boxfill, bg=1) regression.run(canvas, "test_vcs_missing_colorname.png")
import cdms2, os, sys, cdat_info pth = os.path.dirname(os.path.abspath(__file__)) if cdat_info.CDMS_INCLUDE_DAP == 'yes': print 'Testing Dap' f = cdms2.open( 'http://esgcet.llnl.gov/dap/ipcc4/20c3m/ncar_ccsm3_0/pcmdi.ipcc4.ncar_ccsm3_0.20c3m.run6.atm.mo.xml' ) print f.listvariables() else: print 'CDMS not built with DAP support, skipped DAP test' if cdat_info.CDMS_INCLUDE_PP == 'yes': print 'Testing PP' f = cdms2.open(os.path.join(pth, 'testpp.pp')) print f.listvariables() else: print 'CDMS not built with PP support, skipped PP test' if cdat_info.CDMS_INCLUDE_HDF == 'yes': print 'Testing HDF' f = cdms2.open(os.path.join(pth, 'tdata.hdf')) print f.listvariables() else: print 'CDMS not built with HDF4 support, skipped HDF4 test' #if cdat_info.CDMS_INCLUDE_DRS=='yes': # print 'Testing DRS' # f=cdms2.open(os.path.join(cdms2.__path__[0],'..','..','..','..','sample_data','ta_300_850_PCM_O_mm_xy_wa_r0000_0000.dic')) # print f.listvariables() #else:
data_type = DataType.STRUCTURED if data_type == DataType.UNSTRUCTURED: proc_specs = subprocess.check_output('ps').split('\n') for proc_spec in proc_specs: if 'UVIS_DV3D' in proc_spec or 'uvcdat' in proc_spec: pid = int(proc_spec.split()[0]) if pid <> os.getpid(): os.kill(pid, signal.SIGKILL) print "Killing proc: ", proc_spec testDataDir = '/Users/tpmaxwel/Data' datasetPath = os.path.join(testDataDir, 'WRF', 'wrfout_d03_2013-07-02_02-00-00.nc') f = cdms2.open(datasetPath) varname = "U" else: dataDir1 = "/Developer/Data/AConaty/comp-ECMWF" datasetPath = os.path.join(dataDir1, "geos5.xml") f = cdms2.open(datasetPath) varname = "uwnd" # print "Reading variable %s in dataset %s " % ( varname, datasetPath ) # f = cdms2.open( os.path.join( sys.prefix, "sample_data", "geos5-sample.nc") ) # u = f["uwnd"] # dv3d = vcs.create3d_scalar('hoffmuller','xyt')
# Adapted for numpy/ma/cdms2 by convertcdms.py import os, sys import cdms2 as cdms, vcs import EzTemplate # Open file and retrieve data variable. fname = os.path.join(sys.prefix, 'sample_data/clt.nc') cfile = cdms.open(fname) data = cfile('clt') # Initialize vcs. x = vcs.init() # Configure the template. M = EzTemplate.Multi(rows=4, columns=3) M.spacing.horizontal = .25 M.spacing.vertical = .1 # Plot data. for i in range(12): t = M.get() x.plot(data[i], t)
#!/usr/bin/env python # Adapted for numpy/ma/cdms2 by convertcdms.py import cdutil, cdat_info import cdms2 as cdms, vcs, sys, os bg = 0 f = cdms.open( os.path.join(cdat_info.get_prefix(), 'sample_data', 'vertical.nc')) Ps = f('PS') U = f('U') B = f('hybm') A = f('hyam') Po = f('variable_2') P = cdutil.reconstructPressureFromHybrid(Ps, A, B, Po) U2 = cdutil.logLinearInterpolation(U, P) #x=vcs.init() #x.plot(U2,bg=bg) #raw_input()
def handle(infile, tables, user_input_path): """ Transform E3SM.SOILWATER_10CM into CMIP.mrsos float QINTR(time, lat, lon) ; QINTR:long_name = "interception" ; QINTR:units = "mm/s" ; QINTR:cell_methods = "time: mean" ; QINTR:_FillValue = 1.e+36f ; QINTR:missing_value = 1.e+36f ; QINTR:cell_measures = "area: area" ; """ msg = f'Starting {__name__} with {infile}' logging.info(msg) print_message(msg, 'ok') # extract data from the input file f = cdms2.open(infile) veg = f('QVEGT') lat = veg.getLatitude()[:] lon = veg.getLongitude()[:] lat_bnds = f('lat_bnds') lon_bnds = f('lon_bnds') time = veg.getTime() time_bnds = f('time_bounds') f.close() f = cdms2.open(infile.replace('QVEGT', 'QSOIL')) soil = f('QSOIL') f.close() # setup cmor logfile = os.path.join(os.getcwd(), 'logs') if not os.path.exists(logfile): os.makedirs(logfile) _, tail = os.path.split(infile) logfile = os.path.join(logfile, tail.replace('.nc', '.log')) cmor.setup( inpath=tables, netcdf_file_action=cmor.CMOR_REPLACE, logfile=logfile) cmor.dataset_json(user_input_path) table = 'CMIP6_Lmon.json' try: cmor.load_table(table) except: raise Exception('Unable to load table from {}'.format(__name__)) # create axes axes = [{ 'table_entry': 'time', 'units': time.units }, { 'table_entry': 'latitude', 'units': 'degrees_north', 'coord_vals': lat[:], 'cell_bounds': lat_bnds[:] }, { 'table_entry': 'longitude', 'units': 'degrees_east', 'coord_vals': lon[:], 'cell_bounds': lon_bnds[:] }] axis_ids = list() for axis in axes: axis_id = cmor.axis(**axis) axis_ids.append(axis_id) # create the cmor variable varid = cmor.variable('tran', 'kg m-2 s-1', axis_ids, positive='up') # write out the data try: for index, val in enumerate(veg.getTime()[:]): data = veg[index, :] + soil[index, :] cmor.write( varid, data, time_vals=val, time_bnds=[time_bnds[index, :]]) except: raise finally: cmor.close(varid) return 'QVEGT'
['** Known problem file, skipping annual calculation for ', l]) continue elif ('MIROC4h' in l): print "".join(['** MIROC4h file, nyrs set to 5 (not 25) for ', l]) nyrs = 5 elif ('MPI-ESM-MR' in l): print "".join(['** MPI-ESM-MR file, nyrs set to 10 (not 25) for ', l]) nyrs = 10 else: nyrs = 25 #10 ok, 25 CCSM4.historical.r2i1p1.mo.so.ver-v20120409 fails, mergeTime=duplicate/overlapping time issue print "".join([str(nyrs), 'yr annual calculation for ', l]) #continue # Open file and get times f = cdm.open(os.path.join(indir, l)) # Validate that data is online - also consider cdms_filemap/directory combination for existing files try: d = f[var][0, 0, 0, 0] # Assuming 4D variable get single data value del (d) gc.collect() except: print "".join(['** DATA NOT ONLINE: ', l, ' **']) writeToLog(logfile, "".join(['** DATA NOT ONLINE: ', l, ' **'])) continue t = f[var].getTime() c = t.asComponentTime() # Extract info from filename mod = split(l, '.')[1] run = split(l, '.')[3]
def test_test2(self): """ Apply the laplacian finite difference operator to clt """ f = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc', 'r') cltVar = f['clt'] # global sizes nLat, nLon = cltVar.shape[1:] # local rank and number of procs rk = MPI.COMM_WORLD.Get_rank() sz = MPI.COMM_WORLD.Get_size() # compute domain decomposition decomp = CubeDecomp(sz, (nLat, nLon)) # number of processors along each axis npLat, npLon = None, None # list of slices slab = decomp.getSlab(rk) dc = decomp.getDecomp() if dc is not None: npLat, npLon = dc if rk == 0: print '' print 'lat x lon sizes : %d x %d ' % (nLat, nLon) print 'lat x lon domain decomp: %d x %d ' % (npLat, npLon) else: if rk == 0: print 'No uniform decomp could be found for %d procs' % sz sys.exit(1) # starting/ending indices for local domain iLatBeg, iLatEnd = slab[0].start, slab[0].stop iLonBeg, iLonEnd = slab[1].start, slab[1].stop # now read local domain data for time 0 clt = cltVar[0, iLatBeg:iLatEnd, iLonBeg:iLonEnd] # make halo available to other procs numGhosts = 1 clt.exposeHalo(ghostWidth=numGhosts) # compute the star Laplacian in the interior, this does not require # any communication # laplacian = 4*clt[j, i] - clt[j+1, i] - clt[j-1, i] - clt[j, i+1] - clt[j, i-1] # data domain laplaceClt = 4 * clt[:] # local neighbor contributions, no communication laplaceClt[1:, :] -= clt[0:-1, :] laplaceClt[0:-1, :] -= clt[1:, :] laplaceClt[:, 1:] -= clt[:, 0:-1] laplaceClt[:, 0:-1] -= clt[:, 1:] # now compute and fill in the halo # find the procs to the north, east, south, and west. This call will # return None if there is no neighbor. noProc = decomp.getNeighborProc(rk, (1, 0), periodic=(False, True)) soProc = decomp.getNeighborProc(rk, (-1, 0), periodic=(False, True)) eaProc = decomp.getNeighborProc(rk, (0, 1), periodic=(False, True)) weProc = decomp.getNeighborProc(rk, (0, -1), periodic=(False, True)) # correct at north/south poles where zero flux condition applies if noProc is None: laplaceClt[-1, :] -= clt[-1, :] if soProc is None: laplaceClt[0, :] -= clt[0, :] # fetch the remote data in the halo of the neighboring processor. When # the first argument is None, this amounts to a no-op (zero data are # returned. Note that side refers to the neighbour domain. For instance, # the data to the west of the local domain correspond to the east halo # on the neighbouring processor. weCltData = clt.fetchHaloData(weProc, side=(0, +1)) eaCltData = clt.fetchHaloData(eaProc, side=(0, -1)) soCltData = clt.fetchHaloData(soProc, side=(+1, 0)) noCltData = clt.fetchHaloData(noProc, side=(-1, 0)) # finish the operator weSlc = clt.getHaloEllipsis(side=(0, -1)) eaSlc = clt.getHaloEllipsis(side=(0, +1)) soSlc = clt.getHaloEllipsis(side=(-1, 0)) noSlc = clt.getHaloEllipsis(side=(+1, 0)) laplaceClt[weSlc] -= weCltData laplaceClt[eaSlc] -= eaCltData if soProc is not None: laplaceClt[soSlc] -= soCltData if noProc is not None: laplaceClt[noSlc] -= noCltData if True: laplaceClt0 = numpy.zeros(cltVar.shape[1:], cltVar.dtype) # gather the data on proc 0 laplaceClt0List = MPI.COMM_WORLD.gather(laplaceClt, root=0) checksum = 0 if rk == 0: for proc in range(sz): slab = decomp.getSlab(proc) iLatBeg, iLatEnd = slab[0].start, slab[0].stop iLonBeg, iLonEnd = slab[1].start, slab[1].stop laplaceClt0[iLatBeg:iLatEnd, iLonBeg:iLonEnd] = \ laplaceClt0List[proc] checksum = laplaceClt0.sum() print 'checksum = %20.15g' % checksum if False: # plot lat = cltVar.getLatitude() lon = cltVar.getLongitude() latmin, latmax = lat[:].min(), lat[:].max() lonmin, lonmax = lon[:].min(), lon[:].max() mp = Basemap(llcrnrlat=latmin, urcrnrlat=latmax, llcrnrlon=lonmin, urcrnrlon=lonmax, projection='cyl', resolution='l') mp.pcolor(lon[:], lat[:], laplaceClt0) mp.colorbar() mp.drawcoastlines() pylab.title('Laplacian of clt') # checks self.assertLess(abs(checksum), 1.e-3) # clean up clt.freeHalo()
'defcenter' ]: for a in [ 'height', 'angle', 'path', 'halign', 'valign' ]: setattr(tmpo, a, orig[a]) tmpt = self.canvas.gettexttable(orig['Tt_name']) if tmpt.name != 'default': for a in ['spacing', 'expansion', 'color', 'font']: setattr(tmpt, a, orig[a]) self.text = self.canvas.gettext(tmpt.name, tmpo.name) self.setgui() self.orig = [] self.save_vals() except Exception, err: print 'Error:', err, '--------------------------------------' if __name__ == '__main__': import vcs, cdms2 as cdms x = vcs.init() tt = x.gettext('std', '7left') tt.string = ['Hello'] tt.x = [.5] tt.y = [.5] f = cdms.open('/pcmdi/obs/mo/tas/rnl_ncep/tas.rnl_ncep.ctl') s = f('tas', time=slice(0, 1)) x.plot(s) gui = create(canvas=x, table='std', orientation='7left') gui.gui_parent.mainloop()
#!/usr/bin/env python import cdms2, os, sys, vcs, vcs.testing.regression as regression f = cdms2.open(sys.argv[2]) a = f("Z3") x = regression.init() x.setantialiasing(0) x.drawlogooff() x.setbgoutputdimensions(1200, 900, units="pixels") p = x.getprojection("lambert") b = x.createboxfill() b.projection = p x.setbgoutputdimensions(1200, 1091, units="pixels") x.plot(a(latitude=(20, 60), longitude=(-160, -120)), b, bg=1) fileName = os.path.basename(__file__) fileName = os.path.splitext(fileName)[0] fileName += '.png' ret = regression.run(x, fileName)
# Adapted for numpy/ma/cdms2 by convertcdms.py # Import modules: # cdms - Climate Data Management system accesses gridded data. # vcs - Visualization and control System 1D and 2D plotting routines. # os - Operation System routines for Mac, DOS, NT, or Posix depending on the # system you're on. # sys - This module provides access to some objects used or maintained by the # interpreter and to functions that interact strongly with the interpreter. import cdms2 as cdms, vcs, os, sys # Open data file: filepath = os.path.join(vcs.prefix, 'sample_data/clt.nc') cdmsfile = cdms.open(filepath) cdmsfile.listvariables() data = cdmsfile('clt') # Initial VCS: v = vcs.init() # Plot data using the default boxfill graphics method: v.plot(data) # Plot data using the isofill graphics method: v.clear() v.isofill(data) # Plot data using the isoline graphics method: v.clear() v.isoline(data) # Plot data using the boxfill graphics method:
# -*- coding: utf8 -*- import pylab as P # Récupération d'une bathy sous la forme d'une variable cdat import cdms2 cdms2.axis.latitude_aliases.append('y') cdms2.axis.longitude_aliases.append('x') from vacumm.config import data_sample f = cdms2.open(data_sample('ETOPO2v2g_flt.grd')) var = f('z', lon=(-6.1, -3), lat=(47.8, 48.8)) f.close() # Création de l'objet de bathy grillee from vacumm.bathy.bathy import GriddedBathy bathy = GriddedBathy(var) # On définit le traît de côte pour le masquage bathy.set_shoreline('f') # On récupère une version masquée bathy_orig = bathy.bathy(mask=False) bathy_masked = bathy.bathy(mask=True) # Definition d'une grille zoom from vacumm.misc.grid import create_grid new_grid = create_grid((-5.5, -4.6, 0.009), (48.25, 48.6, .006)) # On interpole la bathy masquée sur une autre grille interp_bathy_masked = bathy.regrid(new_grid)
import sys, os import vcs import sys import cdms2 import vtk import os import MV2 bg = not False x = vcs.init() x.setcolormap("rainbow") gm = vcs.createisofill() p = vcs.createprojection() ptype = int('-3') p.type = ptype gm.projection = p xtra = {} xtra["latitude"] = (-90.0, 0.0) f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'clt.nc')) s = f("clt", **xtra) x.plot(s, gm, bg=bg) x.png('test_vcs_basic_isofill_-3_proj_SH.png')
#!/usr/bin/env python import cdms2, numpy, os, sys from markError import NTIME, NLAT, NLON, x, clearError, markError, reportError clearError() print 'Test 4: CdmsFile [numpy.ma] read/write ...', time = numpy.ma.array([0.0, 366.0, 731.0]) lat = numpy.ma.arange(NLAT) * (180. / (NLAT - 1)) - 90. lon = numpy.ma.arange(NLON) * (360.0 / NLON) timestr = ['2000', '2001', '2002'] u = x[0] f = cdms2.createDataset('readwrite.nc') h = cdms2.open(os.path.join(sys.prefix, 'sample_data', 'readonly.nc')) tobj = f.createAxis('time', numpy.ma.array([time[1]])) tobj.units = 'days since 2000-1-1' latobj = f.createAxis('latitude', lat) latobj.units = 'degrees_north' lonobj = f.createAxis('longitude', lon) lonobj.units = 'degrees_east' var = f.createVariable('u', numpy.float, (tobj, latobj, lonobj)) var.units = 'm/s' try: var[:] = u[0] except: markError("Setting a slice") try: var.assignValue(u[0]) except:
] # It is really easy to convert this 2 dimensional list into an array # that CDAT will be able to process further for statistical analysis or simply to disply it my_array=MV2.array(my_data) print my_array.shape # Done, that was easy, wasn't it ? # Now we can fine tune this array, since it has been created with default values # First its name my_array.id='My_Array' # Second it's type, since everytihng in the list was integer it is of type integer # But we can change this my_array=my_array.astype('f') # Here we changed the typecode to 'f' which is float # Accessible values are 'f': float, 'd': double, 'i': int , 'l': long # Ok now we are adding "descriptive" attributes that would be useful to remember my_array.history='first i created a list and then converted to MV2 and changed name and type' # At this point we should also change the axis but this is for another tutorials # We will quickly save it into a new file for future use f=cdms2.open('results.nc','w') f.write(my_array) f.close()
#Reanalysis 2 plot_title = 'MERRA zonal averaged DJF U winds' ########################### END of SETUP ############### #start timing import time start = time.time() import cdms2, vcs, genutil, cdutil # Import cdms2 import cdms2, cdutil import vcs, EzTemplate, sys print "Plotting " + plot_title cdmsfile = cdms2.open(pth1) cl = cdmsfile(varin) #Set Bounds For Monthly Dava cdutil.times.setTimeBoundsMonthly(cl) ##Calculate Climatology DJF for this case cl_djfclimatology = cdutil.times.DJF.climatology(cl) #Calculate zonal average cl_djfclimatology = cdutil.averager(cl_djfclimatology, axis=3, weights='equal') print "Done with calculation, begin plotting" ## PLotting set up from here on down: x = vcs.init() cl_djfclimatology = cl_djfclimatology(squeeze=1)
import sys, os import vcs import sys import cdms2 import vtk import os import MV2 bg = not False x = vcs.init() x.setcolormap("rainbow") gm = vcs.createmeshfill() p = vcs.createprojection() ptype = int('0') p.type = ptype gm.projection = p xtra = {} gm.datawc_y1 = -90.0 gm.datawc_y2 = 0.0 xtra["latitude"] = (-90.0, 0.0) f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'sampleCurveGrid4.nc')) s = f("sample", **xtra) gm.mesh = True x.plot(s, gm, bg=bg) x.png('test_vcs_basic_meshfill_0_proj_SH_via_gm.png')
def dump(self, restart_file=None): """Dump the current instance to a netcdf file""" # File netcdf4() if restart_file is None: restart_file = self.restart_file if restart_file is None: restart_file = self.default_restart_file self.restart_file = restart_file if os.path.exists(restart_file): os.remove(restart_file) f = cdms2.open(restart_file, 'w') # Config # - what was initially asked and some more for sname in self.all_stats + ('sum', 'sqr', 'prod', 'stats'): for st in 'st': value = getattr(self, st + sname) if value is None: continue if isinstance(value, bool): value = int(value) setattr(f, st + sname, value) # - current status f.iterindex = self.iterindex f.nitems = int(self.nitems) f.withtime = -1 if self.withtime is None else int(self.withtime) if self.withtime and self.lasttime: f.lasttime = str(self.lasttime) if self.bins is None: f.bin_edges = 0 else: f.bin_edges = self.bins if f.nitems == 0: # Still no data f.close() return # - already had some data f.dual = int(self.dual) f.ns = self.ns f.nt = self.nt f.nts = self._nts f.tstats = int(self.tstats) f.sstats = int(self.sstats) # Spatial statistics if self.sstats: # Time axes if self.withtime: taxes = () for i, tt in enumerate(self._stimes): taxis = MV2_axisConcatenate(tt) taxis.stataccum_oldid = taxis.id taxis.id = 't' + str(i) taxes += taxis, else: taxes = [cdms2.createAxis(N.arange(self.nt), id='t')] # Count self._dump_array_(f, var=self._scount, id='scount', axis=taxes[0]) # Other stats for key, stats in self._sstats.items(): if isinstance(stats, tuple): for i, stat in enumerate(stats): self._dump_array_(f, var=stat, id='s%s%s' % (key, str(i)), axis=taxes[i]) else: self._dump_array_(f, var=stats, id='s' + key, axis=taxes[0]) # Temporal statistics if self.tstats: # Main axis axis = cdms2.createAxis(N.arange(self.ns), id='s') # Count self._dump_array_(f, var=self._tcount, id='tcount', axis=axis) # Other stats for key in self._dual_accums + self._single_accums: if not getattr(self, 't' + key): continue value = getattr(self, '_t' + key) if key in self._dual_accums: self._dump_array_(f, var=value, id='t' + key, axis=axis) else: for i, stat in enumerate(value): self._dump_array_(f, var=stat, id='t%s%s' % (key, str(i)), axis=axis) # Templates ttemplates = [self._ttemplates] # if self.thist: # ttemplates.append(self._thtemplates) for ttpls in ttemplates: for i, ttpl in enumerate(ttpls): # ttpl = ttpl.clone(copyData=0) # for ia, axis in enumerate(ttpl.getAxisList()): #FIXME:grid cloning # ttpl.setAxis(ia, axis.clone(copyData=0)) _add_id_prefix_(ttpl, 'var%i_' % i, exc=self._baxis) f.write(ttpl) _rm_id_prefix_(ttpl, 'var%i_' % i, exc=self._baxis) # Attributes for ivar, atts in enumerate(self._atts): var = MV2.zeros(1) set_atts(var, atts) var.id = 'var%i_atts' % ivar var.stataccum_id = atts['id'] var.getAxis(0).id = 'var_att_axis' f.write(var) # self._ttemplates, self._thtemplates, self._atts, self._tbase f.close() return f.id
## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by #!/usr/bin/env python import cdms2, numpy, cdtime, os, sys from cdms2 import MV2 as MV from markError import NTIME, NLAT, NLON, x, clearError, markError, reportError clearError() print 'Test 5: get/sub, time functions ...', f = cdms2.open(os.path.join(sys.prefix, 'sample_data', 'test.xml')) v = f.variables['v'] vp = x[1, 1:, 4:12, 8:24] wp = x[1, 2, 4:12] xp = numpy.ma.concatenate((x[1, 1:, 4:12, 8:NLON], x[1, 1:, 4:12, 0:8]), axis=2) # getRegion - positional s = v.getRegion((366., 731., 'ccn'), (-42., 42., 'ccn'), (90., 270., 'con')) if not numpy.ma.allequal(vp, s): markError('getRegion/positional failed') # getRegion - keyword s = v.getRegion(latitude=(-42., 42., 'ccn'), longitude=(90., 270., 'con'), time=(366., 731., 'ccn')) if not numpy.ma.allequal(vp, s): markError('getRegion/keyword failed') # getRegion - wraparound s4 = v.getRegion(time=(366., 731., 'ccn'),
# Adapted for numpy/ma/cdms2 by convertcdms.py import sys, cdms2 as cdms, vcs_legacy, cdtime, support, os bg = support.bg t0 = cdtime.comptime(1987, 8) t1 = cdtime.comptime(1987, 12) f = cdms.open( os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'sample_data', 'ta_ncep_87-6-88-4.nc')) s = f('ta', latitude=slice(5, 6), level=slice(0, 1), longitude=slice(6, 7), squeeze=1) s2 = s() t2 = s2.getTime() t2.units = 'months since 1949-2' x = vcs_legacy.init() y = vcs_legacy.init() b = x.createyxvsx('new2') b.datawc_x1 = t0 b.datawc_x2 = t1 x.plot(s, b, bg=bg) support.check_plot(x) y.plot(s2, b, bg=bg) support.check_plot(y) x.clear()
def load(self, restart_file=None, iterindex=None, nowtime=None): """Load the current instance from a netcdf file :Params: - **restart_file**, optional: Netcdf restart file. - **iterindex**, optional: If given, the restart file is not loaded if ``iterindex`` is greater or equal to the file's ``iterindex`` attribute. - **nowtime**, optional: If given, the restart file is not loaded if ``nowtime`` is greater or equal to the file's ``lasttime`` attribute. """ # File if restart_file is None: restart_file = self.restart_file if restart_file is None: restart_file = self.default_restart_file self.restart_file = restart_file f = cdms2.open(restart_file) # Config # - check status if iterindex is not None: self.iterindex = iterindex if hasattr(self, 'iterindex') and f.iterindex <= self.iterindex: return -1 if nowtime is not None: self.lasttime = comptime(nowtime) if (hasattr(self, 'lasttime') and f.withtime > 0 and self.lasttime and comptime(f.lasttime) <= comptime(self.lasttime)): return -1 # - what was initially asked and some more for sname in self.all_stats + ('sum', 'sqr', 'prod', 'stats'): for st in 'st': if not hasattr(f, st + sname): continue value = getattr(f, st + sname) setattr(self, st + sname, bool(value)) # - current status self.iterindex = int(f.iterindex) self.nitems = int(f.nitems) if f.withtime == -1: self.withtime = None else: self.withtime = bool(f.withtime) if f.withtime: self.lasttime = cdtime.s2c(f.lasttime) if N.isscalar(f.bin_edges): self.bins = None else: self.bins = N.asarray(f.bin_edges) self.nbins = self.bins.shape[0] - 1 self._baxis = f.getAxis('hbin').clone() if self.nitems == 0: # Still no data f.close() return 0 # - already had some data self.dual = bool(f.dual) self.ns = int(f.ns) self.nt = int(f.nt) self._nts = f.nts.tolist() self.tstats = bool(f.tstats) self.sstats = bool(f.sstats) if not self.withtime: self._stimes = None # Spatial statistics if self.sstats: # Time axes if self.withtime: self._stimes = tuple([[] for i in xrange(self.nitems)]) for i, tt in enumerate(self._stimes): taxis = f.getAxis('t' + str(i)) tvalues = self._aslist_(taxis[:]) oldid = taxis.stataccum_oldid for tvals in tvalues: tx = create_time(tvals, taxis.units, id=oldid) cp_atts(taxis, tx, id=False, exclude=[oldid]) self._stimes[i].append(tx) # Count self._scount = self._load_array_(f, id='scount') # Other stats self._sstats = {} for key in self.single_stats: if not self.dual: # single var vid = 's' + key if vid not in f.variables: continue self._sstats[key] = self._load_array_(f, vid), else: # two vars for i in xrange(self.nitems): vid = 's%s%s' % (key, str(i)) if vid not in f.variables: break self._sstats.setdefault(key, ()) self._sstats[key] += self._load_array_(f, vid), for key in self.dual_stats: vid = 's%s' % key if vid in f.variables: self._sstats[key] = self._load_array_(f, vid) # Temporal statistics if self.tstats: # Count self._tcount = self._load_array_(f, 'tcount') # Other stats for key in self._dual_accums + self._single_accums: tid = 't' + key if not getattr(self, tid): continue if key in self._dual_accums: value = self._load_array_(f, tid) setattr(self, '_' + tid, value) else: value = () for i in xrange(self.nitems): value += self._load_array_(f, tid + str(i)), setattr(self, '_' + tid, value) # Templates # - base arrays self._tbase = N.zeros(self.ns) if self.thist: self._thbase = N.zeros((self.nbins, self.ns), 'l') # - cdat templates self._ttemplates = () self._thtemplates = None if self.thist: self._thtemplates = () for i in xrange(self.nitems): prefix = 'var%i_' % i for vname in f.variables: if vname.startswith(prefix) and vname != prefix + 'atts': break ttpl = f(vname) _rm_id_prefix_(ttpl, 'var%i_' % i, exc=self._baxis) self._ttemplates += ttpl, if self.thist: self._thtemplates += self._template_t2ht_(ttpl), # Attributes self._atts = () for ivar in xrange(self.nitems): attrs = f['var%i_atts' % ivar].attributes.copy() attrs['id'] = attrs['stataccum_id'] del attrs['stataccum_id'] self._atts += attrs, f.close() return self.iterindex