def doTCHP(thetaFile, thetaVar, rhoFile, rhoVar, nodata, thresholdTemp, outFile, outFormat, options):
print '# open file'
fhTheta = cdms2.open(thetaFile)
if fhTheta is None:
exitMessage("Could not open file {0}. Exit 2.".format(thetaFile), 2)
fhRho = cdms2.open(rhoFile)
if fhRho is None:
exitMessage("Could not open file {0}; Exit(2).".format(rhoFile), 2)
thetao = fhTheta.variables[thetaVar][:] # [time, levels, lat, lon]
rho = fhRho.variables[rhoVar][:]
levelsTmp = fhTheta.variables['lev_bnds'][:]
levels = numpy.ravel(0.5*(levelsTmp[:,0] + levelsTmp[:,1] ))
print '# mapHeat : time, lat, lon'
mapHeat = numpy.zeros( (thetao.shape[0], thetao.shape[2], thetao.shape[3]) ) - 1
print '# Compressing loops...'
timelatlon=[]
for itime in range(thetao.shape[0]):
for ilat in range(thetao.shape[2]):
for ilon in range(thetao.shape[3]):
if thetao[itime, 0, ilat, ilon] < nodata:
timelatlon.append((itime, ilat, ilon))
# loop over time, lat and lon
print "parsing the maps"
counter=0
for ill in timelatlon:
profileTheta = thetao[ ill[0], : ,ill[1], ill[2] ].ravel()
profileRho = rho[ ill[0], : ,ill[1], ill[2] ].ravel()
heat = 0
heat = computeHeatPotential( profileTheta, profileRho, levels, thresholdTemp, nodata )
mapHeat[ ill[0], ill[1], ill[2] ] = heat
#gdal.TermProgress_nocb( counter/float(len(timelatlon)) )
counter = counter+1
#gdal.TermProgress_nocb(1)
# save result
# outDrv = gdal.GetDriverByName(outformat)
# outDS = outDrv.Create(outFile, mapHeat.shape[2], mapHeat.shape[1], mapHeat.shape[0], GDT_Float32, options)
# outDS.SetProjection(latlon())
output=cdms2.open(outfile, 'w')
output.write(mapHeat)
output.close()
fhTheta.close()
fhRho.close()
# for itime in range(thetao.shape[0]):
# print '.',
# data = numpy.ravel(mapHeat[itime, :, :])
# outDS.GetRasterBand(itime+1).WriteArray( numpy.flipud( data.reshape((mapHeat.shape[1], mapHeat.shape[2])) ) )
# gdal.TermProgress_nocb( itime/float(thetao.shape[0]) )
# gdal.TermProgress_nocb(1)
print
outDS = None
def setUp(self):
self.rootPe = 0
self.pe = MPI.COMM_WORLD.Get_rank()
self.clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, ...]
# self.tas = cdms2.open(sys.prefix + \
# 'tas_ccsr-95a_1979.01-1979.12.nc')('tas')[0, 0,...]
self.tas = cdms2.open(sys.prefix + \
'/sample_data/tas_ecm_1979.nc')('tas')[0, ...]
if PLOT:
lllat = self.clt.getLatitude()[:].min()
urlat = self.clt.getLatitude()[:].max()
lllon = self.clt.getLongitude()[:].min()
urlon = self.clt.getLongitude()[:].max()
self.cmap = bm(llcrnrlat = lllat, llcrnrlon = lllon,
urcrnrlat = urlat, urcrnrlon = urlon,
resolution = 'i', projection = 'cyl')
lllat = self.tas.getLatitude()[:].min()
urlat = self.tas.getLatitude()[:].max()
lllon = self.tas.getLongitude()[:].min()
urlon = self.tas.getLongitude()[:].max()
self.tmap = bm(llcrnrlat = lllat, llcrnrlon = lllon,
urcrnrlat = urlat, urcrnrlon = urlon,
resolution = 'i', projection = 'cyl')
def regrid_5by5(rawOrAnomaly='Anomaly', filteredOrNot='Unfiltered'):
"""
Written By : Arulalan.T
Date : 22.07.2013
"""
inpath = os.path.join(processfilesPath, rawOrAnomaly, filteredOrNot)
for subName in os.listdir(inpath):
anopath = os.path.join(inpath, subName)
for anofile in os.listdir(anopath):
anofilename = anofile.split('.')[0]
varName = anofilename.split('_')[0]
if not anofilename.endswith('_5x5'):
anofilename += '_5x5'
# end of if not anofilename.endswith('_5x5'):
anoFilePath = os.path.join(anopath, anofile)
outfile = anofilename + '.nc'
outpath = os.path.join(anopath, outfile)
if os.path.isfile(outpath):
print "5x5 regridded file already exists for", outpath
continue
# end of if os.path.isfile(outpath):
grid5x5file = os.path.join(__curDir__, _5x5_ncfilename)
varName5x5 = _5x5_ncfilename.split('.')[0]
f = cdms2.open(grid5x5file)
grid5x5 = f[varName5x5].getGrid()
inf = cdms2.open(anoFilePath)
grid_infile = inf[varName].getGrid()
# Regridding the anomaly data
# Creating the horizontal lat,lon regrid
# Note that 'grid_infile' is the source and 'grid5x5' is the target
regridfunc = Horizontal(grid_infile, grid5x5)
anomalytime = inf[varName].getTime().asComponentTime()
print "The out path is ", outpath
loopCount = len(anomalytime)
preview = 0
for day in anomalytime:
if __Debug__:
print "The Date : ", day
data_5x5_regridded = regridfunc(inf(varName, time=day))
outf = cdms2.open(outpath, 'a')
outf.write(data_5x5_regridded)
outf.close()
# make memory free
del data_5x5_regridded
if __showStatusBar:
preview = statusbar(anomalytime.index(day), total=loopCount,
title='Regridding', prev=preview)
# end of if __showStatusBar:
# end of for day in anomalytime:
print
print "The 5x5 regrid for the variable '%s' is stored %s" % (varName, outpath)
inf.close()
f.close()
def do_avg(infile, inpath, variable, nodata, outfile):
# for netcdf3: set flags to 0
cdms2.setNetcdfShuffleFlag(1)
cdms2.setNetcdfDeflateFlag(1)
cdms2.setNetcdfDeflateLevelFlag(3)
# note that this version will erase data whereever a nodata is found in the series
avg=None
nodatamask = None
for ifile in infile:
fname = os.path.join(inpath, ifile)
if not os.path.exists(fname): messageOnExit('file {0} not found on path {1}. Exit(100).'.format(ifile, path), 100)
thisfile = cdms2.open(fname, 'r')
if avg is None:
avg = numpy.array(thisfile[variable][:])
nodatamask = avg >= nodata
else:
avg = avg + numpy.array(thisfile[variable][:])
thisfile.close()
avg = avg/len(infile)
if nodatamask.any():
avg[nodatamask] = nodata
if os.path.exists(outfile): os.remove(outfile)
outfh = cdms2.open(outfile, 'w')
outvar=cdms2.createVariable(avg, typecode='f', id=variable, fill_value=nodata )
outfh.write(outvar)
outfh.close()
def test4_cdmsRegrid(self):
srcF = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
srcGridMask = numpy.array((so == so.missing_value) , numpy.int32)
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')
# create regrid object
r = cdms2.CdmsRegrid(so.getGrid(), clt.getGrid(),
dtype=so.dtype,
regridMethod='conserve',
regridTool='esmf',
srcGridMask=srcGridMask,
srcGridAreas=None,
dstGridMask=None,
dstGridAreas=None)
dstData = r(so)
# checks
if self.pe == 0:
dstDataMask = (dstData == so.missing_value)
print 'number of masked values = ', dstDataMask.sum()
self.assertTrue(str(type(dstData)), str(type(clt)))
dstData.mask = (dstData == so.missing_value)
dstDataMin = dstData.min()
dstDataMax = dstData.max()
zeroValCnt = (dstData == 0).sum()
print 'Number of zero valued cells', zeroValCnt
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertLess(dstDataMax, so.max())
def setUp(self):
filename = "so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc"
h=cdms2.open('./' + filename)
self.hso = h('so')[0, 0, ...]
self.hGrid = [self.hso.getLatitude(), self.hso.getLongitude()]
self.grid2D = self.hso.getGrid()
filename = "clt.nc"
f=cdms2.open(filename)
self.fclt = f('clt')
self.fGrid = [self.fclt.getLatitude(), self.fclt.getLongitude()]
g2D = self.fclt.getGrid().toCurveGrid()
self.grid2D = [g2D.getLatitude()[:], g2D.getLongitude()[:]]
filename = "era40_tas_sample.nc"
g=cdms2.open(filename)
self.gtas = g('tas')
self.gGrid = [self.gtas.getLatitude(), self.gtas.getLongitude()]
g2D = self.gtas.getGrid().toCurveGrid()
self.grid2D = [g2D.getLatitude()[:], g2D.getLongitude()[:]]
self.gcltVal010101 = 74.0087890625
self.ftasVal1010 = 283.39410400390625
self.soVal1010 = 33.88642120361328
self.MissingValue = 1.e20
self.eps = 1e-4
def XXtest5_regrid(self):
srcF = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')
dstData = so.regrid(clt.getGrid(),
regridTool = 'esmf',
regridMethod='conserve')
if self.pe == 0:
dstDataMask = (dstData == so.missing_value)
dstDataFltd = dstData * (1 - dstDataMask)
zeroValCnt = (dstData == 0).sum()
if so.missing_value > 0:
dstDataMin = dstData.min()
dstDataMax = dstDataFltd.max()
else:
dstDataMin = dstDataFltd.min()
dstDataMax = dstData.max()
zeroValCnt = (dstData == 0).sum()
print 'Number of zero valued cells', zeroValCnt
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertLess(dstDataMax, so.max())
if False:
pylab.figure(1)
pylab.pcolor(so, vmin=20, vmax=40)
pylab.colorbar()
pylab.title('so')
pylab.figure(2)
pylab.pcolor(dstData, vmin=20, vmax=40)
pylab.colorbar()
pylab.title('dstData')
def test_pack(c):
for pack in [False, True]:
# print "Testing pack:",pack
f = cdms2.open("test_pack.nc", "w")
f.write(c, id="packed", pack=pack)
f.close()
f = cdms2.open("test_pack.nc")
d = f("packed")
f.close()
assert c.dtype == d.dtype
if pack in [True, numpy.int16]:
atol = rtol = 1.0 / pow(2, 11)
# elif pack in [numpy.int,numpy.int64]:
# atol=rtol = 1./pow(2,63)
# elif pack == numpy.int32:
# atol=rtol = 1./pow(2,31)
# elif pack == numpy.int8:
# atol=rtol = 1./pow(2,7)
elif pack is False:
atol = rtol = 0.0
# print "M:",pack,c.max(),d.max()
assert numpy.ma.allclose(numpy.ma.maximum(c), numpy.ma.maximum(d), True, rtol, atol)
# print "m:",pack,c.min(),d.min()
assert numpy.ma.allclose(numpy.ma.minimum(c), numpy.ma.minimum(d), True, rtol, atol)
# print "D:",pack, (c-d).max(),(c-d).min(),((c-d)/c).max(),rtol
assert numpy.ma.allclose(c, d, True, rtol, atol)
os.remove("test_pack.nc")
def download_file(filename, varlist, proxy_cert):
outfilename="/tmp/" + str(time.time())
myvarlist=varlist.split(',')
if len(myvarlist)>0 and myvarlist[0] != '':
try:
f=cdms2.open(filename)
fout=cdms2.open(outfilename,'w')
for var in myvarlist:
myvar=str(var)
fout.write(f(myvar,time=slice(0,1)))
fout.close()
f.close()
return outfilename
except Exception as e:
print e
return None
else:
try:
wget_url="http://pcmdi9.llnl.gov/esg-search/wget?url="+filename
print wget_url
with open(outfilename,'w') as outfile:
flock(outfile,LOCK_EX)
cmd="wget --certificate %s -t 2 -T 10 --private-key %s -O %s %s"%(proxy_cert,proxy_cert,outfilename,wget_url)
output=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,shell=True)
time.sleep(.1)
flock(outfile,LOCK_UN)
outfile.close()
return outfilename
except Exception as e:
return None
def closeness( self, varname, filename, rtol, atol ):
#pdb.set_trace()
testfname = os.path.join( self.outpath, filename )
#print '>>>>>>>>>>>>>>>>>>> ', testfname
baselinefname = os.path.join( self.baselinepath, filename )
#print '>>>>>>>>>>>>>>>>>>> ', baselinefname
f = cdms2.open( testfname )
g = cdms2.open( baselinefname )
try:
fvar = f(varname)
except:
print varname, ' is not in file ', testname
sys.exit(1)
try:
gvar = g(varname)
except:
print varname, ' is not in file ', baselinefname
sys.exit(1)
#print '>>>>>>>>>>>>>>>>>>> fvar', fvar.shape
#print '>>>>>>>>>>>>>>>>>>> gvar', gvar.shape
close = numpy.ma.allclose( fvar, gvar, rtol=rtol, atol=atol )
if close:
print "fvar and gvar are close for", varname
else:
print "fvar and gvar differ for", varname
print "max difference", (fvar-gvar).max()
print "min difference", (fvar-gvar).min()
f.close()
g.close()
return close
def setUp(self):
"""
Set up the grids to pass to mvGenericRegrid
"""
self.doPlots = False
self.clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, ...]
# Convert to curvilinear
cds, cds_forBounds = [], []
cds.append(self.clt.getLatitude())
cds.append(self.clt.getLongitude())
cds_forBounds.append(self.clt.getLatitude())
cds_forBounds.append(self.clt.getLongitude())
self.cltGrid, nDims = regrid2.gsRegrid.makeCurvilinear(cds)
self.cltInterp = numpy.array(self.clt) * 0.0 + self.clt.missing_value
self.cltInterpInterp = numpy.array(self.clt) * 0.0 + self.clt.missing_value
# Salinity check
f = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
self.so = f('so')
soGrid = []
soGrid.append(self.so.getLatitude())
soGrid.append(self.so.getLongitude())
self.soGrid = soGrid
self.soInterp = numpy.array(self.clt) * 0.0 + self.so.missing_value
self.tol = 1e2
self.comm = MPI.COMM_WORLD
self.rank = self.comm.Get_rank()
self.size = self.comm.Get_size()
def initialize( self, args, **cfg_args ):
self.configure( **cfg_args )
( grid_file, data_file, interface, varname, grid_coords, var_proc_op ) = args
self.gf = cdms2.open( grid_file ) if grid_file else None
self.df = cdms2.open( data_file )
self.var = self.getProcessedVariable( varname, var_proc_op )
self.extractMetadata()
self.grid = self.var.getGrid()
self.lev = self.getLevel(self.var)
lon, lat = self.getLatLon( grid_coords )
if ( id(lon) <> id(None) ) and ( id(lat) <> id(None) ):
self.time = self.var.getTime()
z_scale = 0.5
self.missing_value = self.var.attributes.get( 'missing_value', None )
if self.lev == None:
domain = self.var.getDomain()
for axis in domain:
if PlotType.isLevelAxis( axis[0].id.lower() ):
self.lev = axis[0]
break
self.computePoints()
self.setPointHeights( height_var=grid_coords[3], z_scale=z_scale )
np_var_data_block = self.getDataBlock(self.var).flatten()
if self.missing_value: var_data = numpy.ma.masked_equal( np_var_data_block, self.missing_value, False )
else: var_data = np_var_data_block
self.point_data_arrays[ self.defvar ] = var_data
self.vrange = ( var_data.min(), var_data.max() )
self.var_data_cache[ self.iTimeStep ] = var_data
self.metadata['vbounds'] = self.vrange
def coarsenVars( filen, newfilen ):
"""Reads a NetCDF file, filen, and writes a new file, newfilen, containing all its variables,
but regridded in its horizontal axeseb to a 12x14 (15 degree) lat-lon grid.
"""
latbnds = numpy.linspace(-90,90,nlat)
lat = 0.5*( latbnds[1:]+latbnds[:-1] )
lataxis=cdms2.createAxis( lat, id='lat' )
lonbnds = numpy.linspace(0,360,nlon)
lon = 0.5*( lonbnds[1:]+lonbnds[:-1] )
lonaxis=cdms2.createAxis( lon, id='lon' )
newgrid=cdms2.createRectGrid(lataxis, lonaxis)
f = cdms2.open(filen)
nf = cdms2.open(newfilen,'w')
for varn in f.variables.keys():
print "working on",varn
var = f(varn)
if var.getGrid() is None:
print varn,"has no grid!"
continue
newvar = var.regrid(newgrid) # works even if var has levels
newvar.id = var.id # before, id was var.id+'_CdmaRegrid'
nf.write(newvar)
for attr in f.attributes.keys():
setattr(nf,attr,getattr(f,attr))
if hasattr(nf,'history'):
nf.history = nf.history + '; some variables deleted'
f.close()
nf.close()
def getData(self, variable=None):
'''
Aggreate all data from the file list in the order given.
'''
filename = self.flist[0].strip()
f = cdms2.open( filename, 'r' )
if( variable != None ):
self.vartoread = variable
else:
self.vartoread = self.variable.id
data = f(self.vartoread)[:]
# ---------------------------
# Concatenate following files
# ---------------------------
for filename in self.flist[ 1: ]:
print "reading %s" % filename.strip()
f = cdms2.open( filename.strip(), 'r' )
data2 = f(self.vartoread)[:]
data = numpy.concatenate((data,data2), axis=0)
f.close()
data=ma.array(data=data, \
fill_value=self.missing_value, \
copy=0, \
dtype='float32' )
data=ma.masked_equal(data, self.missing_value, copy=0)
return data
def test2_varRegrid(self):
print
print 'test2_varRegrid'
srcF = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')
diag = {'srcAreas': None, 'dstAreas': None,
'srcAreaFractions': None, 'dstAreaFractions': None}
soInterp = so.regrid(clt.getGrid(),
regridTool = 'esmf',
regridMethod='conserve',
diag = diag)
if self.pe == 0:
totSrcArea = diag['srcAreas'].sum()
totDstArea = diag['dstAreas'].sum()
totSrcFrac = diag['srcAreaFractions'].sum()
self.assertEqual(numpy.isnan(totSrcFrac).sum(), 0)
self.assertLess(abs(totSrcArea - 4*pi)/(4*pi), 0.02)
self.assertLess(abs(totDstArea - 4*pi)/(4*pi), 0.01)
soMass = (so*diag['srcAreas']).sum()
inMass = (soInterp*diag['dstAreas']).sum()
print soMass, inMass
diff = abs(soMass - inMass)/soMass
self.assertLess(diff, 7.e-7)
if False:
pylab.subplot(1, 2, 1)
pylab.pcolor(so, vmin = 20, vmax = 40)
pylab.colorbar()
pylab.title('so')
pylab.subplot(1, 2, 2)
pylab.pcolor(soInterp, vmin = 20, vmax = 40)
pylab.colorbar()
pylab.title('soInterp')
def test_2d_esmf_conserv(self):
print 'running test_2d_esmf_conserv...'
f = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = f('so')[0, 0, :, :]
clt = cdms2.open(sys.prefix + '/sample_data/clt.nc')('clt')[0, :, :]
tic = time.time()
soInterp = so.regrid(clt.getGrid(), regridTool='ESMF', regridMethod='Conservative')
soInterpInterp = soInterp.regrid(so.getGrid(), regridTool='ESMF',
regridMethod='Conservative')
toc = time.time()
print 'time to interpolate (ESMF conservative) forward/backward: ', toc - tic
ntot = reduce(operator.mul, so.shape)
avgdiff = numpy.sum(so - soInterpInterp) / float(ntot)
print 'avgdiff = ', avgdiff
if PLOT:
pylab.figure(2)
pylab.subplot(2, 2, 1)
pylab.pcolor(so, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: so')
pylab.subplot(2, 2, 2)
pylab.pcolor(soInterp, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: soInterp')
pylab.subplot(2, 2, 3)
pylab.pcolor(soInterpInterp, vmin=20.0, vmax=40.0)
pylab.colorbar()
pylab.title('ESMF conserve regrid: soInterpInterp')
pylab.subplot(2, 2, 4)
pylab.pcolor(so - soInterpInterp, vmin=-0.5, vmax=0.5)
pylab.colorbar()
pylab.title('ESMF conserve regrid: error')
def demoScene(self): from sys import executable from os.path import split,join import cdms2 sample_dir = join(split(executable)[0],"..","sample_data") mean = cdms2.open(join(sample_dir,"mean_1900_1_1900_12.nc"))['tas'] std = cdms2.open(join(sample_dir,"std_1900_1_1900_12.nc"))['tas'] cloudiness = cdms2.open(join(sample_dir,"cl_A1.20C3M_1.CCSM.atmm.1900-01_cat_1909-12.nc"))['cl'] mean = mean(longitude=(-180,180)) std = std(longitude=(-180,180)) mean.getTime().toRelativeTime(cloudiness.getTime().units) std.getTime().toRelativeTime(cloudiness.getTime().units) t = mean.getTime() t = [t[0],t[-1]] cloudiness = cloudiness(longitude=(-180,180),time=t) scene = VisusMovieScene.MovieScene(mean, std, cloudiness) pyvisus.gVCDATScenes += [scene]
def test4_cdmsRegrid(self):
srcF = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')
# create regrid object
r = cdms2.CdmsRegrid(so.getGrid(), clt.getGrid(), so.dtype,
regridMethod='linear', regridTool='libcf',
srcGridMask=(so == so.missing_value),
srcGridAreas=None,
dstGridMask=None, dstGridAreas=None,
mkCyclic=False, handleCut=False)
dstData = r(so)
# checks
dstDataMask = (dstData == so.missing_value)
dstDataFltd = dstData * (1 - dstDataMask)
if so.missing_value > 0:
dstDataMin = dstData.min()
dstDataMax = dstDataFltd.max()
else:
dstDataMin = dstDataFltd.min()
dstDataMax = dstData.max()
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertGreater(dstDataMin, so.min())
self.assertLess(dstDataMax, so.max())
def Xtest1_gsRegrid(self):
srcF = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')
srcGrd = [so.getGrid().getLatitude(), so.getGrid().getLongitude()]
dstGrd = [clt.getGrid().getLatitude(), clt.getGrid().getLongitude()]
# create regrid object
r = regrid2.gsRegrid.Regrid(srcGrd, dstGrd, src_bounds=None,
mkCyclic=False, handleCut=False)
# set mask
r.setMask( (so == so.missing_value) )
# compute interpolation weights
r.computeWeights(nitermax=20, tolpos=0.01)
# make sure most of the weights have ben found
print 'ratio of valid over number of nodes: %f' % (r.getNumValid()/float(r.getNumDstPoints()))
self.assertGreater(r.getNumValid(), 0)
# create dst data container
dstShp = r.getDstGrid()[0].shape
print 'dst data shape: ', dstShp
dstData = numpy.ones(dstShp, so.dtype) * so.missing_value
# interpolate
r.apply(so, dstData)
# checks
dstDataMask = (dstData == so.missing_value)
dstDataFltd = dstData * (1 - dstDataMask)
if so.missing_value > 0:
dstDataMin = dstData.min()
dstDataMax = dstDataFltd.max()
else:
dstDataMin = dstDataFltd.min()
dstDataMax = dstData.max()
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertGreater(dstDataMin, so.min())
self.assertLess(dstDataMax, so.max())
def Xtest3_genericRegrid(self):
srcF = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
so = srcF('so')[0, 0, ...]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')
srcGrd = [so.getGrid().getLatitude(), so.getGrid().getLongitude()]
dstGrd = [clt.getGrid().getLatitude(), clt.getGrid().getLongitude()]
# create regrid object
r = regrid2.mvGenericRegrid.GenericRegrid(srcGrd, dstGrd, so.dtype,
regridMethod='linear', regridTool='libcf',
srcGridMask=(so == so.missing_value), srcBounds=None, srcGridAreas=None,
dstGridMask=None, dstBounds=None, dstGridAreas=None,
mkCyclic=False, handleCut=False)
# compute weights
r.computeWeights(nitermax=20, tolpos=0.01)
# create dst data container
dstShp = r.getDstGrid()[0].shape
dstData = numpy.ones(dstShp, so.dtype) * so.missing_value
# interpolate
r.apply(so, dstData)
# checks
dstDataMask = (dstData == so.missing_value)
dstDataFltd = dstData * (1 - dstDataMask)
if so.missing_value > 0:
dstDataMin = dstData.min()
dstDataMax = dstDataFltd.max()
else:
dstDataMin = dstDataFltd.min()
dstDataMax = dstData.max()
print 'min/max value of dstData: %f %f' % (dstDataMin, dstDataMax)
self.assertGreater(dstDataMin, so.min())
self.assertLess(dstDataMax, so.max())
def setUp(self):
self.so = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')('so')
self.tasGood = cdms2.open(cdat_info.get_sampledata_path() + \
'/tas_Amon_HadGEM2-A_amip_r1i2p1_197809-200811_2timesteps.nc')('tas')
self.tasBad = cdms2.open(cdat_info.get_sampledata_path() + \
'/tas_Amon_ACCESS1-0_historical_r1i1p1_185001-189912_2timesteps.nc')('tas')
def test_test2(self):
"""
2D ESMP
"""
u = cdms2.open(sys.prefix + "/sample_data/clt.nc")("u")[0, 0, ...]
clt = cdms2.open(sys.prefix + "/sample_data/clt.nc")("clt")[0, ...]
ctlOnUGrid = clt.regrid(u.getGrid(), regridTool="ESMP")
# print 'ctlOnUGrid.getGrid() = ', type(ctlOnUGrid.getGrid())
self.assertRegexpMatches(str(type(ctlOnUGrid.getGrid())), "cdms2.grid.TransientRectGrid")
def setUp(self):
fso = cdms2.open(sys.prefix + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
self.soLevel = fso('so')[0,0:5,...]
self.soTime = fso('so')[:,0,...]
self.so = fso('so')[:,0:5,...]
fclt = cdms2.open(sys.prefix + 'sample_data/clt.nc')
self.clt = fclt('clt')
def setUp(self):
fso = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
self.soLevel = fso('so')[0,0:5,...]
self.soTime = fso('so')[:,0,...]
self.so = fso('so')[:,0:5,...]
fclt = cdms2.open(cdat_info.get_sampledata_path() + "/clt.nc")
self.clt = fclt('clt')
def initialize( self, args, **cfg_args ):
self.configure( **cfg_args )
( grid_file, data_file, interface, grd_varnames, grd_coords, var_proc_op, ROI, subSpace ) = args
self.interface = interface
self.roi = ROI
self.gf = cdms2.open( grid_file ) if grid_file else None
self.df = cdms2.open( data_file )
self.grid_varname = grd_varnames[0] if ( grd_varnames <> None ) else self.df.variables[0]
self.grid_coords = grd_coords
self.initPoints( var_proc_op )
def setUp(self):
f = cdms2.open('so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
self.accessModel = f('so')
h = cdms2.open('so_Omon_HadGEM2-CC_historical_r1i1p1_185912-186911_2timesteps.nc')
self.hadGEM2Model = h('so')
self.clt = cdms2.open('clt.nc')('clt')[0, :, :]
self.eps = 1e-5
def test_test2(self):
"""
2D ESMP
"""
u = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('u')[0, 0,...]
clt = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc')('clt')[0, ...]
ctlOnUGrid = clt.regrid( u.getGrid(), regridTool = "ESMP" )
#print 'ctlOnUGrid.getGrid() = ', type(ctlOnUGrid.getGrid())
self.assertRegexpMatches(str(type(ctlOnUGrid.getGrid())),
"cdms2.grid.TransientRectGrid")
def setUp(self):
"""
Set up the grids to pass to mvGenericRegrid
"""
f = cdms2.open(cdat_info.get_prefix() + \
'/sample_data/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
self.so = f('so')
g = cdms2.open(cdat_info.get_prefix() + '/sample_data/clt.nc')
self.clt = g('clt')
def loadFileFromURL(self,url):
## let's figure out between dap or local
if url[:7].lower()=="http://":
f=cdms2.open(str(url))
elif url[:7]=="file://":
f=cdms2.open(str(url[6:]))
else:
# can't figure it out skipping
f=None
return f
def setUp(self):
access = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_ACCESS1-0_historical_r1i1p1_185001-185412_2timesteps.nc')
self.access = access('so')[0, 0, ...]
self.tasGood = cdms2.open(cdat_info.get_sampledata_path() + \
'/tas_Amon_HadGEM2-A_amip_r1i2p1_197809-200811_2timesteps.nc')('tas')
giss = cdms2.open(cdat_info.get_sampledata_path() + \
'/so_Omon_GISS-E2-R_historicalNat_r5i1p1_185001-187512_2timesteps.nc')
self.giss = giss('so')[0,0,...]
# Adapted for numpy/ma/cdms2 by convertcdms.py
import vcs_legacy, cdms2 as cdms, os, sys, support
bg = support.bg
tp = vcs_legacy.init()
tp.portrait()
fnm = os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'sample_data',
'clt.nc')
f = cdms.open(fnm)
s = f('clt')
tp.plot(s, bg=bg)
support.check_plot(tp)
tit = tp.createtext('tit')
tit.x = [0.5]
tit.y = [.9]
tit.halign = 'center'
tit.height = 20
tit.color = 241
n = 6
i = 0
tit.string = 'Line %d - Decorrelation Time of Volume Mean Temperature Change' % i
tp.plot(tit, bg=bg)
support.check_plot(tp)
tp.mode = 1
for i in range(1, n):
tit.y = [1. - (i / float(n))]
support.check_plot(tp)
import cdms2, sys, numpy, cdat_info
f = cdms2.open(cdat_info.get_sampledata_path() + "/clt.nc")
s = f("clt")
jsn = s.dumps()
s2 = cdms2.createVariable(jsn, fromJSON=True)
assert (numpy.allclose(s2, s))
def compute(params):
fileName = params.fileName
startyear = params.args.firstyear
finalyear = params.args.lastyear
month = params.args.month
monthname = params.monthname
varbname = params.varname
template = populateStringConstructor(args.filename_template, args)
template.variable = varbname
reverted = template.reverse(os.path.basename(fileName))
dataname = reverted["model"]
if dataname not in args.skip:
try:
print('Data source:', dataname)
print('Opening %s ...' % fileName)
f = cdms2.open(fileName)
iYear = 0
dmean = None
for year in range(startyear, finalyear + 1):
print('Year %s:' % year)
startTime = cdtime.comptime(year, month)
# Last possible second to get all tpoints
finishtime = startTime.add(1, cdtime.Month).add(
-1, cdtime.Minute)
print('Reading %s from %s for time interval %s to %s ...' %
(varbname, fileName, startTime, finishtime))
# Transient variable stores data for current year's month.
tvarb = f(varbname, time=(startTime, finishtime, "ccn"))
# *HARD-CODES conversion from kg/m2/sec to mm/day.
tvarb *= 86400
# The following tasks need to be done only once, extracting
# metadata from first-year file:
tc = tvarb.getTime().asComponentTime()
current = tc[0]
while current.month == month:
end = cdtime.comptime(current.year, current.month,
current.day).add(1, cdtime.Day)
sub = tvarb(time=(current, end, "con"))
# Assumes first dimension of input ("axis#0") is time
tmp = numpy.ma.average(sub, axis=0)
sh = list(tmp.shape)
sh.insert(0, 1)
if dmean is None:
dmean = tmp.reshape(sh)
else:
dmean = numpy.ma.concatenate((dmean, tmp.reshape(sh)),
axis=0)
current = end
iYear += 1
f.close()
stdvalues = cdms2.MV2.array(genutil.statistics.std(dmean))
stdvalues.setAxis(0, tvarb.getLatitude())
stdvalues.setAxis(1, tvarb.getLongitude())
stdvalues.id = 'dailySD'
# Standard deviation has same units as mean.
stdvalues.units = "mm/d"
stdoutfile = ('%s_%s_%s_%s-%s_std_of_dailymeans.nc') % (
varbname, dataname, monthname, str(startyear), str(finalyear))
except Exception as err:
print("Failed for model: %s with error: %s" % (dataname, err))
if not os.path.exists(args.results_dir):
os.makedirs(args.results_dir)
g = cdms2.open(os.path.join(args.results_dir, stdoutfile), 'w')
g.write(stdvalues)
g.close()
#
#
############################################################################
# #
# Import: VCS and cdms modules. #
# #
############################################################################
import vcs_legacy
import cdms2 as cdms
import sys
import os
import support
f = cdms.open(
os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'sample_data',
'meshfill.nc'))
M = f('Mesh')
s = f('Data')
x = vcs_legacy.init()
x.scriptrun(
os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'bin', 'ASD.scr'))
m = x.getmeshfill('ASD')
m.wrap = [0, 360]
m.mesh = 'y'
#mn,mx=vcs_legacy.minmax(s)
levs = vcs_legacy.mkscale(-10, 30)
#print levs
levs = vcs_legacy.mkevenlevels(levs[0], levs[-1], 256)
levs2 = levs[::25]
import os
from durolib import globalAttWrite
import cdms2 as cdm
import numpy as np
import MV2 as mv
# Set netcdf file criterion - turned on from default 0s
cdm.setCompressionWarnings(0) # Suppress warnings
cdm.setNetcdfShuffleFlag(0)
cdm.setNetcdfDeflateFlag(1)
cdm.setNetcdfDeflateLevelFlag(9)
cdm.setAutoBounds(1)
# JRA25
infile = '/work/durack1/Shared/obs_data/pcmdi_metrics/197901/anl_land25.ctl'
f_h = cdm.open(infile)
soilwhbl = f_h('soilwhbl')[0, 0, ...]
landMask = soilwhbl.mask
lat = soilwhbl.getLatitude()
lon = soilwhbl.getLongitude()
# Convert to percentage
landMask = landMask * 100 # Convert to boolean fraction
# Rename
landMask = cdm.createVariable(
landMask,
id='sftlf',
axes=soilwhbl.getAxisList(),
typecode='float32')
landMask.original_name = "soilwhbl"
def compute(params):
fileName = params.fileName
month = params.args.month
monthname = params.monthname
varbname = params.varname
template = populateStringConstructor(args.filename_template, args)
template.variable = varbname
# Units on output (*may be converted below from the units of input*)
outunits = 'mm/d'
startime = 1.5 # GMT value for starting time-of-day
dataname = params.args.model
if dataname is None or dataname.find("*") != -1:
# model not passed or passed as *
reverted = template.reverse(os.path.basename(fileName))
print("REVERYING", reverted, dataname)
dataname = reverted["model"]
if dataname not in args.skip:
try:
print('Data source:', dataname)
print('Opening %s ...' % fileName)
f = cdms2.open(fileName)
# Composite-mean and composite-s.d diurnal cycle for month and year(s):
iYear = 0
for year in range(args.firstyear, args.lastyear + 1):
print('Year %s:' % year)
startTime = cdtime.comptime(year, month)
# Last possible second to get all tpoints
finishtime = startTime.add(1, cdtime.Month).add(
-1, cdtime.Minute)
print('Reading %s from %s for time interval %s to %s ...' %
(varbname, fileName, startTime, finishtime))
# Transient variable stores data for current year's month.
tvarb = f(varbname, time=(startTime, finishtime))
# *HARD-CODES conversion from kg/m2/sec to mm/day.
tvarb *= 86400
print('Shape:', tvarb.shape)
# The following tasks need to be done only once, extracting
# metadata from first-year file:
if year == args.firstyear:
tc = tvarb.getTime().asComponentTime()
print("DATA FROM:", tc[0], "to", tc[-1])
day1 = cdtime.comptime(tc[0].year, tc[0].month)
day1 = tc[0]
firstday = tvarb(time=(day1, day1.add(1., cdtime.Day),
"con"))
dimensions = firstday.shape
print(' Shape = ', dimensions)
# Number of time points in the selected month for one year
N = dimensions[0]
nlats = dimensions[1]
nlons = dimensions[2]
deltaH = 24. / N
dayspermo = tvarb.shape[0] // N
print(
' %d timepoints per day, %d hr intervals between timepoints'
% (N, deltaH))
comptime = firstday.getTime()
modellons = tvarb.getLongitude()
modellats = tvarb.getLatitude()
# Longitude values are needed later to compute Local Solar
# Times.
lons = modellons[:]
print(' Creating temporary storage and output fields ...')
# Sorts tvarb into separate GMTs for one year
tvslice = MV2.zeros((N, dayspermo, nlats, nlons))
# Concatenates tvslice over all years
concatenation = MV2.zeros(
(N, dayspermo * nYears, nlats, nlons))
LSTs = MV2.zeros((N, nlats, nlons))
for iGMT in range(N):
hour = iGMT * deltaH + startime
print(
' Computing Local Standard Times for GMT %5.2f ...'
% hour)
for j in range(nlats):
for k in range(nlons):
LSTs[iGMT, j, k] = (hour + lons[k] / 15) % 24
for iGMT in range(N):
hour = iGMT * deltaH + startime
print(' Choosing timepoints with GMT %5.2f ...' % hour)
print("days per mo :", dayspermo)
# Transient-variable slice: every Nth tpoint gets all of
# the current GMT's tpoints for current year:
tvslice[iGMT] = tvarb[iGMT::N]
concatenation[iGMT, iYear * dayspermo:(iYear + 1) *
dayspermo] = tvslice[iGMT]
iYear += 1
f.close()
# For each GMT, take mean and standard deviation over all years for
# the chosen month:
avgvalues = MV2.zeros((N, nlats, nlons))
stdvalues = MV2.zeros((N, nlats, nlons))
for iGMT in range(N):
hour = iGMT * deltaH + startime
print(
'Computing mean and standard deviation over all GMT %5.2f timepoints ...'
% hour)
# Assumes first dimension of input ("axis#0") is time
avgvalues[iGMT] = MV2.average(concatenation[iGMT], axis=0)
stdvalues[iGMT] = genutil.statistics.std(concatenation[iGMT])
avgvalues.id = 'diurnalmean'
stdvalues.id = 'diurnalstd'
LSTs.id = 'LST'
avgvalues.units = outunits
# Standard deviation has same units as mean (not so for
# higher-moment stats).
stdvalues.units = outunits
LSTs.units = 'hr'
LSTs.longname = 'Local Solar Time'
avgvalues.setAxis(0, comptime)
avgvalues.setAxis(1, modellats)
avgvalues.setAxis(2, modellons)
stdvalues.setAxis(0, comptime)
stdvalues.setAxis(1, modellats)
stdvalues.setAxis(2, modellons)
LSTs.setAxis(0, comptime)
LSTs.setAxis(1, modellats)
LSTs.setAxis(2, modellons)
avgoutfile = ('%s_%s_%s_%s-%s_diurnal_avg.nc') % (
varbname, dataname, monthname, str(
args.firstyear), str(args.lastyear))
stdoutfile = ('%s_%s_%s_%s-%s_diurnal_std.nc') % (
varbname, dataname, monthname, str(
args.firstyear), str(args.lastyear))
LSToutfile = ('%s_%s_LocalSolarTimes.nc' % (varbname, dataname))
if not os.path.exists(args.results_dir):
os.makedirs(args.results_dir)
f = cdms2.open(os.path.join(args.results_dir, avgoutfile), 'w')
g = cdms2.open(os.path.join(args.results_dir, stdoutfile), 'w')
h = cdms2.open(os.path.join(args.results_dir, LSToutfile), 'w')
f.write(avgvalues)
g.write(stdvalues)
h.write(LSTs)
f.close()
g.close()
h.close()
except Exception as err:
print("Failed for model %s with erro: %s" % (dataname, err))
# Adapted for numpy/ma/cdms2 by convertcdms.py
import cdms2, sys, ZonalMeans
cdms2.setAutoBounds("on")
cdms2.axis.level_aliases.append('depth')
import vcs
import vcs.test.support
bg = vcs.test.support.bg
x = vcs.init()
# Open data file
f = cdms2.open('test_data.nc')
O2 = f("O2", level=slice(0, 5))
print O2.shape, O2.mask
print 'Level:', O2.getLevel()
f.close()
# Open grid file
f = cdms2.open('test_grid.nc')
# 1st with just bounds
blon = f('bounds_lon')
blat = f('bounds_lat')
#mask = 1 - f('mask')
area = f('area')
res = ZonalMeans.compute(O2,
bounds_lon=blon,
bounds_lat=blat,
area=area,
delta_band=5)
print res[1].shape
x.plot(res[1][:, 0], bg=bg)
def Xtest_test1(self):
"""
Test case with domain decomposition
"""
f = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc', 'r')
cltVar = f['clt']
# global sizes
nLat, nLon = cltVar.shape[1:]
# local rank and number of procs
rk = MPI.COMM_WORLD.Get_rank()
sz = MPI.COMM_WORLD.Get_size()
# compute domain decomposition
npLat, npLon = getDomainDecomp(sz, (nLat, nLon))
if rk == 0:
print ''
print 'lat x lon sizes : %d x %d ' % (nLat, nLon)
print 'lat x lon domain decomp: %d x %d ' % (npLat, npLon)
# number of points in each local domain
nlat, nlon = nLat / npLat, nLon / npLon
ipLat = rk // npLon
ipLon = rk % npLon
# starting/ending indices for local domain
iLatBeg, iLatEnd = ipLat * nlat, (ipLat + 1) * nlat
iLonBeg, iLonEnd = ipLon * nlon, (ipLon + 1) * nlon
# now read local domain data for time 0
clt = cltVar[0, iLatBeg:iLatEnd, iLonBeg:iLonEnd]
# make halo available to other procs
numGhosts = 1
clt.exposeHalo(ghostWidth=numGhosts)
# find the procs to the north, east, south, and west
noProc = min(ipLat + 1, npLat - 1) * npLon + ipLon + 0
soProc = max(ipLat - 1, 0) * npLon + ipLon + 0
# warp around
eaProc = (ipLat + 0) * npLon + (ipLon + 1) % npLon
weProc = (ipLat + 0) * npLon + (ipLon - 1) % npLon
# fetch the remote data
noCltData = clt.fetchHaloData(noProc, side=(-1, 0))
soCltData = clt.fetchHaloData(soProc, side=(+1, 0))
eaCltData = clt.fetchHaloData(eaProc, side=(0, -1))
weCltData = clt.fetchHaloData(weProc, side=(0, +1))
# slabs in local index space
noSlc = clt.getHaloEllipsis(side=(-1, 0))
soSlc = clt.getHaloEllipsis(side=(+1, 0))
eaSlc = clt.getHaloEllipsis(side=(0, -1))
weSlc = clt.getHaloEllipsis(side=(0, +1))
tol = 1.e-10
success = True
if True:
# north
if ipLat < npLat - 1:
print 'ipLat < npLat - 1:', ipLat, npLat - 1
igLatBeg = (ipLat + 1) * nlat
igLatEnd = igLatBeg + numGhosts
igLonBeg = ipLon * nlon
igLonEnd = igLonBeg + nlon
globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd]
if abs(numpy.sum(globalCltData - noCltData)) > tol:
success = False
# south
if ipLat >= 1:
print 'ipLat >= 1:', ipLat, npLat - 1
igLatBeg = (ipLat + 0) * nlat - numGhosts
igLatEnd = igLatBeg + numGhosts
igLonBeg = ipLon * nlon
igLonEnd = igLonBeg + nlon
globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd]
if abs(numpy.sum(globalCltData - soCltData)) > tol:
success = False
# east
igLatBeg = ipLat * nlat
igLatEnd = igLatBeg + nlat
igLonBeg = ((ipLon + 1) % npLon) * nlon
igLonEnd = igLonBeg + numGhosts
globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd]
if abs(numpy.sum(globalCltData - eaCltData)) > tol:
success = False
# west
igLatBeg = ipLat * nlat
igLatEnd = igLatBeg + nlat
igLonBeg = ((ipLon - 1) % npLon) * nlon + nlon - numGhosts
igLonEnd = igLonBeg + numGhosts
globalCltData = cltVar[0, igLatBeg:igLatEnd, igLonBeg:igLonEnd]
if abs(numpy.sum(globalCltData - weCltData)) > tol:
success = False
self.assertEqual(success, True)
clt.freeHalo()
import cdms2
import os
import sys
import vcs
cdmsfile = cdms2.open(os.path.join(vcs.prefix, "sample_data", "clt.nc"))
data = cdmsfile('clt')
x = vcs.init()
t = x.gettemplate('default')
x.plot(data, t, bg=True)
# This should force the image to update
x.setcolormap('bl_to_drkorang')
testFilename = "test_vcs_setcolormap.png"
x.png(testFilename)
import os, sys, cdms2, vcs, vcs.testing.regression as regression
# Load the clt data:
dataFile = cdms2.open(os.path.join(vcs.sample_data, "clt.nc"))
clt = dataFile("clt", slice(0, 1), squeeze=1)
height, width = clt.shape
clt.mask = [[True if i % 2 else False for i in range(width)]
for _ in range(height)]
# Initialize canvas:
canvas = regression.init()
# Create and plot quick boxfill with default settings:
# Only have to test boxfill because all 2D methods use the same code
# for missing values
boxfill = canvas.createboxfill()
# Change the missing color to a colorname
boxfill.missing = "lightgrey"
canvas.plot(clt, boxfill, bg=1)
regression.run(canvas, "test_vcs_missing_colorname.png")
import cdms2, os, sys, cdat_info
pth = os.path.dirname(os.path.abspath(__file__))
if cdat_info.CDMS_INCLUDE_DAP == 'yes':
print 'Testing Dap'
f = cdms2.open(
'http://esgcet.llnl.gov/dap/ipcc4/20c3m/ncar_ccsm3_0/pcmdi.ipcc4.ncar_ccsm3_0.20c3m.run6.atm.mo.xml'
)
print f.listvariables()
else:
print 'CDMS not built with DAP support, skipped DAP test'
if cdat_info.CDMS_INCLUDE_PP == 'yes':
print 'Testing PP'
f = cdms2.open(os.path.join(pth, 'testpp.pp'))
print f.listvariables()
else:
print 'CDMS not built with PP support, skipped PP test'
if cdat_info.CDMS_INCLUDE_HDF == 'yes':
print 'Testing HDF'
f = cdms2.open(os.path.join(pth, 'tdata.hdf'))
print f.listvariables()
else:
print 'CDMS not built with HDF4 support, skipped HDF4 test'
#if cdat_info.CDMS_INCLUDE_DRS=='yes':
# print 'Testing DRS'
# f=cdms2.open(os.path.join(cdms2.__path__[0],'..','..','..','..','sample_data','ta_300_850_PCM_O_mm_xy_wa_r0000_0000.dic'))
# print f.listvariables()
#else:
data_type = DataType.STRUCTURED
if data_type == DataType.UNSTRUCTURED:
proc_specs = subprocess.check_output('ps').split('\n')
for proc_spec in proc_specs:
if 'UVIS_DV3D' in proc_spec or 'uvcdat' in proc_spec:
pid = int(proc_spec.split()[0])
if pid <> os.getpid():
os.kill(pid, signal.SIGKILL)
print "Killing proc: ", proc_spec
testDataDir = '/Users/tpmaxwel/Data'
datasetPath = os.path.join(testDataDir, 'WRF',
'wrfout_d03_2013-07-02_02-00-00.nc')
f = cdms2.open(datasetPath)
varname = "U"
else:
dataDir1 = "/Developer/Data/AConaty/comp-ECMWF"
datasetPath = os.path.join(dataDir1, "geos5.xml")
f = cdms2.open(datasetPath)
varname = "uwnd"
# print "Reading variable %s in dataset %s " % ( varname, datasetPath )
# f = cdms2.open( os.path.join( sys.prefix, "sample_data", "geos5-sample.nc") )
# u = f["uwnd"]
# dv3d = vcs.create3d_scalar('hoffmuller','xyt')
# Adapted for numpy/ma/cdms2 by convertcdms.py
import os, sys
import cdms2 as cdms, vcs
import EzTemplate
# Open file and retrieve data variable.
fname = os.path.join(sys.prefix, 'sample_data/clt.nc')
cfile = cdms.open(fname)
data = cfile('clt')
# Initialize vcs.
x = vcs.init()
# Configure the template.
M = EzTemplate.Multi(rows=4, columns=3)
M.spacing.horizontal = .25
M.spacing.vertical = .1
# Plot data.
for i in range(12):
t = M.get()
x.plot(data[i], t)
#!/usr/bin/env python
# Adapted for numpy/ma/cdms2 by convertcdms.py
import cdutil, cdat_info
import cdms2 as cdms, vcs, sys, os
bg = 0
f = cdms.open(
os.path.join(cdat_info.get_prefix(), 'sample_data', 'vertical.nc'))
Ps = f('PS')
U = f('U')
B = f('hybm')
A = f('hyam')
Po = f('variable_2')
P = cdutil.reconstructPressureFromHybrid(Ps, A, B, Po)
U2 = cdutil.logLinearInterpolation(U, P)
#x=vcs.init()
#x.plot(U2,bg=bg)
#raw_input()
def handle(infile, tables, user_input_path):
"""
Transform E3SM.SOILWATER_10CM into CMIP.mrsos
float QINTR(time, lat, lon) ;
QINTR:long_name = "interception" ;
QINTR:units = "mm/s" ;
QINTR:cell_methods = "time: mean" ;
QINTR:_FillValue = 1.e+36f ;
QINTR:missing_value = 1.e+36f ;
QINTR:cell_measures = "area: area" ;
"""
msg = f'Starting {__name__} with {infile}'
logging.info(msg)
print_message(msg, 'ok')
# extract data from the input file
f = cdms2.open(infile)
veg = f('QVEGT')
lat = veg.getLatitude()[:]
lon = veg.getLongitude()[:]
lat_bnds = f('lat_bnds')
lon_bnds = f('lon_bnds')
time = veg.getTime()
time_bnds = f('time_bounds')
f.close()
f = cdms2.open(infile.replace('QVEGT', 'QSOIL'))
soil = f('QSOIL')
f.close()
# setup cmor
logfile = os.path.join(os.getcwd(), 'logs')
if not os.path.exists(logfile):
os.makedirs(logfile)
_, tail = os.path.split(infile)
logfile = os.path.join(logfile, tail.replace('.nc', '.log'))
cmor.setup(
inpath=tables,
netcdf_file_action=cmor.CMOR_REPLACE,
logfile=logfile)
cmor.dataset_json(user_input_path)
table = 'CMIP6_Lmon.json'
try:
cmor.load_table(table)
except:
raise Exception('Unable to load table from {}'.format(__name__))
# create axes
axes = [{
'table_entry': 'time',
'units': time.units
}, {
'table_entry': 'latitude',
'units': 'degrees_north',
'coord_vals': lat[:],
'cell_bounds': lat_bnds[:]
}, {
'table_entry': 'longitude',
'units': 'degrees_east',
'coord_vals': lon[:],
'cell_bounds': lon_bnds[:]
}]
axis_ids = list()
for axis in axes:
axis_id = cmor.axis(**axis)
axis_ids.append(axis_id)
# create the cmor variable
varid = cmor.variable('tran', 'kg m-2 s-1', axis_ids, positive='up')
# write out the data
try:
for index, val in enumerate(veg.getTime()[:]):
data = veg[index, :] + soil[index, :]
cmor.write(
varid,
data,
time_vals=val,
time_bnds=[time_bnds[index, :]])
except:
raise
finally:
cmor.close(varid)
return 'QVEGT'
['** Known problem file, skipping annual calculation for ', l])
continue
elif ('MIROC4h' in l):
print "".join(['** MIROC4h file, nyrs set to 5 (not 25) for ', l])
nyrs = 5
elif ('MPI-ESM-MR' in l):
print "".join(['** MPI-ESM-MR file, nyrs set to 10 (not 25) for ', l])
nyrs = 10
else:
nyrs = 25
#10 ok, 25 CCSM4.historical.r2i1p1.mo.so.ver-v20120409 fails, mergeTime=duplicate/overlapping time issue
print "".join([str(nyrs), 'yr annual calculation for ', l])
#continue
# Open file and get times
f = cdm.open(os.path.join(indir, l))
# Validate that data is online - also consider cdms_filemap/directory combination for existing files
try:
d = f[var][0, 0, 0, 0]
# Assuming 4D variable get single data value
del (d)
gc.collect()
except:
print "".join(['** DATA NOT ONLINE: ', l, ' **'])
writeToLog(logfile, "".join(['** DATA NOT ONLINE: ', l, ' **']))
continue
t = f[var].getTime()
c = t.asComponentTime()
# Extract info from filename
mod = split(l, '.')[1]
run = split(l, '.')[3]
def test_test2(self):
"""
Apply the laplacian finite difference operator to clt
"""
f = cdms2.open(cdat_info.get_sampledata_path() + '/clt.nc', 'r')
cltVar = f['clt']
# global sizes
nLat, nLon = cltVar.shape[1:]
# local rank and number of procs
rk = MPI.COMM_WORLD.Get_rank()
sz = MPI.COMM_WORLD.Get_size()
# compute domain decomposition
decomp = CubeDecomp(sz, (nLat, nLon))
# number of processors along each axis
npLat, npLon = None, None
# list of slices
slab = decomp.getSlab(rk)
dc = decomp.getDecomp()
if dc is not None:
npLat, npLon = dc
if rk == 0:
print ''
print 'lat x lon sizes : %d x %d ' % (nLat, nLon)
print 'lat x lon domain decomp: %d x %d ' % (npLat, npLon)
else:
if rk == 0:
print 'No uniform decomp could be found for %d procs' % sz
sys.exit(1)
# starting/ending indices for local domain
iLatBeg, iLatEnd = slab[0].start, slab[0].stop
iLonBeg, iLonEnd = slab[1].start, slab[1].stop
# now read local domain data for time 0
clt = cltVar[0, iLatBeg:iLatEnd, iLonBeg:iLonEnd]
# make halo available to other procs
numGhosts = 1
clt.exposeHalo(ghostWidth=numGhosts)
# compute the star Laplacian in the interior, this does not require
# any communication
# laplacian = 4*clt[j, i] - clt[j+1, i] - clt[j-1, i] - clt[j, i+1] - clt[j, i-1]
# data domain
laplaceClt = 4 * clt[:]
# local neighbor contributions, no communication
laplaceClt[1:, :] -= clt[0:-1, :]
laplaceClt[0:-1, :] -= clt[1:, :]
laplaceClt[:, 1:] -= clt[:, 0:-1]
laplaceClt[:, 0:-1] -= clt[:, 1:]
# now compute and fill in the halo
# find the procs to the north, east, south, and west. This call will
# return None if there is no neighbor.
noProc = decomp.getNeighborProc(rk, (1, 0), periodic=(False, True))
soProc = decomp.getNeighborProc(rk, (-1, 0), periodic=(False, True))
eaProc = decomp.getNeighborProc(rk, (0, 1), periodic=(False, True))
weProc = decomp.getNeighborProc(rk, (0, -1), periodic=(False, True))
# correct at north/south poles where zero flux condition applies
if noProc is None:
laplaceClt[-1, :] -= clt[-1, :]
if soProc is None:
laplaceClt[0, :] -= clt[0, :]
# fetch the remote data in the halo of the neighboring processor. When
# the first argument is None, this amounts to a no-op (zero data are
# returned. Note that side refers to the neighbour domain. For instance,
# the data to the west of the local domain correspond to the east halo
# on the neighbouring processor.
weCltData = clt.fetchHaloData(weProc, side=(0, +1))
eaCltData = clt.fetchHaloData(eaProc, side=(0, -1))
soCltData = clt.fetchHaloData(soProc, side=(+1, 0))
noCltData = clt.fetchHaloData(noProc, side=(-1, 0))
# finish the operator
weSlc = clt.getHaloEllipsis(side=(0, -1))
eaSlc = clt.getHaloEllipsis(side=(0, +1))
soSlc = clt.getHaloEllipsis(side=(-1, 0))
noSlc = clt.getHaloEllipsis(side=(+1, 0))
laplaceClt[weSlc] -= weCltData
laplaceClt[eaSlc] -= eaCltData
if soProc is not None:
laplaceClt[soSlc] -= soCltData
if noProc is not None:
laplaceClt[noSlc] -= noCltData
if True:
laplaceClt0 = numpy.zeros(cltVar.shape[1:], cltVar.dtype)
# gather the data on proc 0
laplaceClt0List = MPI.COMM_WORLD.gather(laplaceClt, root=0)
checksum = 0
if rk == 0:
for proc in range(sz):
slab = decomp.getSlab(proc)
iLatBeg, iLatEnd = slab[0].start, slab[0].stop
iLonBeg, iLonEnd = slab[1].start, slab[1].stop
laplaceClt0[iLatBeg:iLatEnd, iLonBeg:iLonEnd] = \
laplaceClt0List[proc]
checksum = laplaceClt0.sum()
print 'checksum = %20.15g' % checksum
if False:
# plot
lat = cltVar.getLatitude()
lon = cltVar.getLongitude()
latmin, latmax = lat[:].min(), lat[:].max()
lonmin, lonmax = lon[:].min(), lon[:].max()
mp = Basemap(llcrnrlat=latmin,
urcrnrlat=latmax,
llcrnrlon=lonmin,
urcrnrlon=lonmax,
projection='cyl',
resolution='l')
mp.pcolor(lon[:], lat[:], laplaceClt0)
mp.colorbar()
mp.drawcoastlines()
pylab.title('Laplacian of clt')
# checks
self.assertLess(abs(checksum), 1.e-3)
# clean up
clt.freeHalo()
'defcenter'
]:
for a in [
'height', 'angle', 'path', 'halign', 'valign'
]:
setattr(tmpo, a, orig[a])
tmpt = self.canvas.gettexttable(orig['Tt_name'])
if tmpt.name != 'default':
for a in ['spacing', 'expansion', 'color', 'font']:
setattr(tmpt, a, orig[a])
self.text = self.canvas.gettext(tmpt.name, tmpo.name)
self.setgui()
self.orig = []
self.save_vals()
except Exception, err:
print 'Error:', err, '--------------------------------------'
if __name__ == '__main__':
import vcs, cdms2 as cdms
x = vcs.init()
tt = x.gettext('std', '7left')
tt.string = ['Hello']
tt.x = [.5]
tt.y = [.5]
f = cdms.open('/pcmdi/obs/mo/tas/rnl_ncep/tas.rnl_ncep.ctl')
s = f('tas', time=slice(0, 1))
x.plot(s)
gui = create(canvas=x, table='std', orientation='7left')
gui.gui_parent.mainloop()
#!/usr/bin/env python
import cdms2, os, sys, vcs, vcs.testing.regression as regression
f = cdms2.open(sys.argv[2])
a = f("Z3")
x = regression.init()
x.setantialiasing(0)
x.drawlogooff()
x.setbgoutputdimensions(1200, 900, units="pixels")
p = x.getprojection("lambert")
b = x.createboxfill()
b.projection = p
x.setbgoutputdimensions(1200, 1091, units="pixels")
x.plot(a(latitude=(20, 60), longitude=(-160, -120)), b, bg=1)
fileName = os.path.basename(__file__)
fileName = os.path.splitext(fileName)[0]
fileName += '.png'
ret = regression.run(x, fileName)
# Adapted for numpy/ma/cdms2 by convertcdms.py
# Import modules:
# cdms - Climate Data Management system accesses gridded data.
# vcs - Visualization and control System 1D and 2D plotting routines.
# os - Operation System routines for Mac, DOS, NT, or Posix depending on the
# system you're on.
# sys - This module provides access to some objects used or maintained by the
# interpreter and to functions that interact strongly with the interpreter.
import cdms2 as cdms, vcs, os, sys
# Open data file:
filepath = os.path.join(vcs.prefix, 'sample_data/clt.nc')
cdmsfile = cdms.open(filepath)
cdmsfile.listvariables()
data = cdmsfile('clt')
# Initial VCS:
v = vcs.init()
# Plot data using the default boxfill graphics method:
v.plot(data)
# Plot data using the isofill graphics method:
v.clear()
v.isofill(data)
# Plot data using the isoline graphics method:
v.clear()
v.isoline(data)
# Plot data using the boxfill graphics method:
# -*- coding: utf8 -*-
import pylab as P
# Récupération d'une bathy sous la forme d'une variable cdat
import cdms2
cdms2.axis.latitude_aliases.append('y')
cdms2.axis.longitude_aliases.append('x')
from vacumm.config import data_sample
f = cdms2.open(data_sample('ETOPO2v2g_flt.grd'))
var = f('z', lon=(-6.1, -3), lat=(47.8, 48.8))
f.close()
# Création de l'objet de bathy grillee
from vacumm.bathy.bathy import GriddedBathy
bathy = GriddedBathy(var)
# On définit le traît de côte pour le masquage
bathy.set_shoreline('f')
# On récupère une version masquée
bathy_orig = bathy.bathy(mask=False)
bathy_masked = bathy.bathy(mask=True)
# Definition d'une grille zoom
from vacumm.misc.grid import create_grid
new_grid = create_grid((-5.5, -4.6, 0.009), (48.25, 48.6, .006))
# On interpole la bathy masquée sur une autre grille
interp_bathy_masked = bathy.regrid(new_grid)
import sys, os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x = vcs.init()
x.setcolormap("rainbow")
gm = vcs.createisofill()
p = vcs.createprojection()
ptype = int('-3')
p.type = ptype
gm.projection = p
xtra = {}
xtra["latitude"] = (-90.0, 0.0)
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'clt.nc'))
s = f("clt", **xtra)
x.plot(s, gm, bg=bg)
x.png('test_vcs_basic_isofill_-3_proj_SH.png')
#!/usr/bin/env python
import cdms2, numpy, os, sys
from markError import NTIME, NLAT, NLON, x, clearError, markError, reportError
clearError()
print 'Test 4: CdmsFile [numpy.ma] read/write ...',
time = numpy.ma.array([0.0, 366.0, 731.0])
lat = numpy.ma.arange(NLAT) * (180. / (NLAT - 1)) - 90.
lon = numpy.ma.arange(NLON) * (360.0 / NLON)
timestr = ['2000', '2001', '2002']
u = x[0]
f = cdms2.createDataset('readwrite.nc')
h = cdms2.open(os.path.join(sys.prefix, 'sample_data', 'readonly.nc'))
tobj = f.createAxis('time', numpy.ma.array([time[1]]))
tobj.units = 'days since 2000-1-1'
latobj = f.createAxis('latitude', lat)
latobj.units = 'degrees_north'
lonobj = f.createAxis('longitude', lon)
lonobj.units = 'degrees_east'
var = f.createVariable('u', numpy.float, (tobj, latobj, lonobj))
var.units = 'm/s'
try:
var[:] = u[0]
except:
markError("Setting a slice")
try:
var.assignValue(u[0])
except:
]
# It is really easy to convert this 2 dimensional list into an array
# that CDAT will be able to process further for statistical analysis or simply to disply it
my_array=MV2.array(my_data)
print my_array.shape
# Done, that was easy, wasn't it ?
# Now we can fine tune this array, since it has been created with default values
# First its name
my_array.id='My_Array'
# Second it's type, since everytihng in the list was integer it is of type integer
# But we can change this
my_array=my_array.astype('f')
# Here we changed the typecode to 'f' which is float
# Accessible values are 'f': float, 'd': double, 'i': int , 'l': long
# Ok now we are adding "descriptive" attributes that would be useful to remember
my_array.history='first i created a list and then converted to MV2 and changed name and type'
# At this point we should also change the axis but this is for another tutorials
# We will quickly save it into a new file for future use
f=cdms2.open('results.nc','w')
f.write(my_array)
f.close()
#Reanalysis 2 plot_title = 'MERRA zonal averaged DJF U winds' ########################### END of SETUP ############### #start timing import time start = time.time() import cdms2, vcs, genutil, cdutil # Import cdms2 import cdms2, cdutil import vcs, EzTemplate, sys print "Plotting " + plot_title cdmsfile = cdms2.open(pth1) cl = cdmsfile(varin) #Set Bounds For Monthly Dava cdutil.times.setTimeBoundsMonthly(cl) ##Calculate Climatology DJF for this case cl_djfclimatology = cdutil.times.DJF.climatology(cl) #Calculate zonal average cl_djfclimatology = cdutil.averager(cl_djfclimatology, axis=3, weights='equal') print "Done with calculation, begin plotting" ## PLotting set up from here on down: x = vcs.init() cl_djfclimatology = cl_djfclimatology(squeeze=1)
import sys, os
import vcs
import sys
import cdms2
import vtk
import os
import MV2
bg = not False
x = vcs.init()
x.setcolormap("rainbow")
gm = vcs.createmeshfill()
p = vcs.createprojection()
ptype = int('0')
p.type = ptype
gm.projection = p
xtra = {}
gm.datawc_y1 = -90.0
gm.datawc_y2 = 0.0
xtra["latitude"] = (-90.0, 0.0)
f = cdms2.open(os.path.join(vcs.prefix, 'sample_data', 'sampleCurveGrid4.nc'))
s = f("sample", **xtra)
gm.mesh = True
x.plot(s, gm, bg=bg)
x.png('test_vcs_basic_meshfill_0_proj_SH_via_gm.png')
def dump(self, restart_file=None):
"""Dump the current instance to a netcdf file"""
# File
netcdf4()
if restart_file is None:
restart_file = self.restart_file
if restart_file is None:
restart_file = self.default_restart_file
self.restart_file = restart_file
if os.path.exists(restart_file):
os.remove(restart_file)
f = cdms2.open(restart_file, 'w')
# Config
# - what was initially asked and some more
for sname in self.all_stats + ('sum', 'sqr', 'prod', 'stats'):
for st in 'st':
value = getattr(self, st + sname)
if value is None: continue
if isinstance(value, bool): value = int(value)
setattr(f, st + sname, value)
# - current status
f.iterindex = self.iterindex
f.nitems = int(self.nitems)
f.withtime = -1 if self.withtime is None else int(self.withtime)
if self.withtime and self.lasttime:
f.lasttime = str(self.lasttime)
if self.bins is None:
f.bin_edges = 0
else:
f.bin_edges = self.bins
if f.nitems == 0: # Still no data
f.close()
return
# - already had some data
f.dual = int(self.dual)
f.ns = self.ns
f.nt = self.nt
f.nts = self._nts
f.tstats = int(self.tstats)
f.sstats = int(self.sstats)
# Spatial statistics
if self.sstats:
# Time axes
if self.withtime:
taxes = ()
for i, tt in enumerate(self._stimes):
taxis = MV2_axisConcatenate(tt)
taxis.stataccum_oldid = taxis.id
taxis.id = 't' + str(i)
taxes += taxis,
else:
taxes = [cdms2.createAxis(N.arange(self.nt), id='t')]
# Count
self._dump_array_(f, var=self._scount, id='scount', axis=taxes[0])
# Other stats
for key, stats in self._sstats.items():
if isinstance(stats, tuple):
for i, stat in enumerate(stats):
self._dump_array_(f,
var=stat,
id='s%s%s' % (key, str(i)),
axis=taxes[i])
else:
self._dump_array_(f,
var=stats,
id='s' + key,
axis=taxes[0])
# Temporal statistics
if self.tstats:
# Main axis
axis = cdms2.createAxis(N.arange(self.ns), id='s')
# Count
self._dump_array_(f, var=self._tcount, id='tcount', axis=axis)
# Other stats
for key in self._dual_accums + self._single_accums:
if not getattr(self, 't' + key): continue
value = getattr(self, '_t' + key)
if key in self._dual_accums:
self._dump_array_(f, var=value, id='t' + key, axis=axis)
else:
for i, stat in enumerate(value):
self._dump_array_(f,
var=stat,
id='t%s%s' % (key, str(i)),
axis=axis)
# Templates
ttemplates = [self._ttemplates]
# if self.thist:
# ttemplates.append(self._thtemplates)
for ttpls in ttemplates:
for i, ttpl in enumerate(ttpls):
# ttpl = ttpl.clone(copyData=0)
# for ia, axis in enumerate(ttpl.getAxisList()): #FIXME:grid cloning
# ttpl.setAxis(ia, axis.clone(copyData=0))
_add_id_prefix_(ttpl, 'var%i_' % i, exc=self._baxis)
f.write(ttpl)
_rm_id_prefix_(ttpl, 'var%i_' % i, exc=self._baxis)
# Attributes
for ivar, atts in enumerate(self._atts):
var = MV2.zeros(1)
set_atts(var, atts)
var.id = 'var%i_atts' % ivar
var.stataccum_id = atts['id']
var.getAxis(0).id = 'var_att_axis'
f.write(var)
# self._ttemplates, self._thtemplates, self._atts, self._tbase
f.close()
return f.id
## Automatically adapted for numpy.oldnumeric Aug 01, 2007 by
#!/usr/bin/env python
import cdms2, numpy, cdtime, os, sys
from cdms2 import MV2 as MV
from markError import NTIME, NLAT, NLON, x, clearError, markError, reportError
clearError()
print 'Test 5: get/sub, time functions ...',
f = cdms2.open(os.path.join(sys.prefix, 'sample_data', 'test.xml'))
v = f.variables['v']
vp = x[1, 1:, 4:12, 8:24]
wp = x[1, 2, 4:12]
xp = numpy.ma.concatenate((x[1, 1:, 4:12, 8:NLON], x[1, 1:, 4:12, 0:8]),
axis=2)
# getRegion - positional
s = v.getRegion((366., 731., 'ccn'), (-42., 42., 'ccn'), (90., 270., 'con'))
if not numpy.ma.allequal(vp, s): markError('getRegion/positional failed')
# getRegion - keyword
s = v.getRegion(latitude=(-42., 42., 'ccn'),
longitude=(90., 270., 'con'),
time=(366., 731., 'ccn'))
if not numpy.ma.allequal(vp, s): markError('getRegion/keyword failed')
# getRegion - wraparound
s4 = v.getRegion(time=(366., 731., 'ccn'),
# Adapted for numpy/ma/cdms2 by convertcdms.py
import sys, cdms2 as cdms, vcs_legacy, cdtime, support, os
bg = support.bg
t0 = cdtime.comptime(1987, 8)
t1 = cdtime.comptime(1987, 12)
f = cdms.open(
os.path.join(cdms.__path__[0], '..', '..', '..', '..', 'sample_data',
'ta_ncep_87-6-88-4.nc'))
s = f('ta',
latitude=slice(5, 6),
level=slice(0, 1),
longitude=slice(6, 7),
squeeze=1)
s2 = s()
t2 = s2.getTime()
t2.units = 'months since 1949-2'
x = vcs_legacy.init()
y = vcs_legacy.init()
b = x.createyxvsx('new2')
b.datawc_x1 = t0
b.datawc_x2 = t1
x.plot(s, b, bg=bg)
support.check_plot(x)
y.plot(s2, b, bg=bg)
support.check_plot(y)
x.clear()
def load(self, restart_file=None, iterindex=None, nowtime=None):
"""Load the current instance from a netcdf file
:Params:
- **restart_file**, optional: Netcdf restart file.
- **iterindex**, optional: If given, the restart file is not loaded if
``iterindex`` is greater or equal to the file's ``iterindex`` attribute.
- **nowtime**, optional: If given, the restart file is not loaded if
``nowtime`` is greater or equal to the file's ``lasttime`` attribute.
"""
# File
if restart_file is None:
restart_file = self.restart_file
if restart_file is None:
restart_file = self.default_restart_file
self.restart_file = restart_file
f = cdms2.open(restart_file)
# Config
# - check status
if iterindex is not None:
self.iterindex = iterindex
if hasattr(self, 'iterindex') and f.iterindex <= self.iterindex:
return -1
if nowtime is not None:
self.lasttime = comptime(nowtime)
if (hasattr(self, 'lasttime') and f.withtime > 0 and self.lasttime
and comptime(f.lasttime) <= comptime(self.lasttime)):
return -1
# - what was initially asked and some more
for sname in self.all_stats + ('sum', 'sqr', 'prod', 'stats'):
for st in 'st':
if not hasattr(f, st + sname): continue
value = getattr(f, st + sname)
setattr(self, st + sname, bool(value))
# - current status
self.iterindex = int(f.iterindex)
self.nitems = int(f.nitems)
if f.withtime == -1:
self.withtime = None
else:
self.withtime = bool(f.withtime)
if f.withtime:
self.lasttime = cdtime.s2c(f.lasttime)
if N.isscalar(f.bin_edges):
self.bins = None
else:
self.bins = N.asarray(f.bin_edges)
self.nbins = self.bins.shape[0] - 1
self._baxis = f.getAxis('hbin').clone()
if self.nitems == 0: # Still no data
f.close()
return 0
# - already had some data
self.dual = bool(f.dual)
self.ns = int(f.ns)
self.nt = int(f.nt)
self._nts = f.nts.tolist()
self.tstats = bool(f.tstats)
self.sstats = bool(f.sstats)
if not self.withtime:
self._stimes = None
# Spatial statistics
if self.sstats:
# Time axes
if self.withtime:
self._stimes = tuple([[] for i in xrange(self.nitems)])
for i, tt in enumerate(self._stimes):
taxis = f.getAxis('t' + str(i))
tvalues = self._aslist_(taxis[:])
oldid = taxis.stataccum_oldid
for tvals in tvalues:
tx = create_time(tvals, taxis.units, id=oldid)
cp_atts(taxis, tx, id=False, exclude=[oldid])
self._stimes[i].append(tx)
# Count
self._scount = self._load_array_(f, id='scount')
# Other stats
self._sstats = {}
for key in self.single_stats:
if not self.dual: # single var
vid = 's' + key
if vid not in f.variables:
continue
self._sstats[key] = self._load_array_(f, vid),
else: # two vars
for i in xrange(self.nitems):
vid = 's%s%s' % (key, str(i))
if vid not in f.variables:
break
self._sstats.setdefault(key, ())
self._sstats[key] += self._load_array_(f, vid),
for key in self.dual_stats:
vid = 's%s' % key
if vid in f.variables:
self._sstats[key] = self._load_array_(f, vid)
# Temporal statistics
if self.tstats:
# Count
self._tcount = self._load_array_(f, 'tcount')
# Other stats
for key in self._dual_accums + self._single_accums:
tid = 't' + key
if not getattr(self, tid): continue
if key in self._dual_accums:
value = self._load_array_(f, tid)
setattr(self, '_' + tid, value)
else:
value = ()
for i in xrange(self.nitems):
value += self._load_array_(f, tid + str(i)),
setattr(self, '_' + tid, value)
# Templates
# - base arrays
self._tbase = N.zeros(self.ns)
if self.thist:
self._thbase = N.zeros((self.nbins, self.ns), 'l')
# - cdat templates
self._ttemplates = ()
self._thtemplates = None
if self.thist:
self._thtemplates = ()
for i in xrange(self.nitems):
prefix = 'var%i_' % i
for vname in f.variables:
if vname.startswith(prefix) and vname != prefix + 'atts':
break
ttpl = f(vname)
_rm_id_prefix_(ttpl, 'var%i_' % i, exc=self._baxis)
self._ttemplates += ttpl,
if self.thist:
self._thtemplates += self._template_t2ht_(ttpl),
# Attributes
self._atts = ()
for ivar in xrange(self.nitems):
attrs = f['var%i_atts' % ivar].attributes.copy()
attrs['id'] = attrs['stataccum_id']
del attrs['stataccum_id']
self._atts += attrs,
f.close()
return self.iterindex
