def writeField(lons, lats, tmes, varname, dataField):
#-----------------------------------------------------------------------------------
#
# purpose: write an output field
#
# usage:
#
# passed : lons, lats, tmes, filename, varname, dataField
#
# returned: None
#
#-----------------------------------------------------------------------------------
fileObj = cdms2.createDataset(varname + '.nc')
lon_axis = fileObj.createAxis('longitude', lons)
lon_axis.units = "degrees_east"
lat_axis = fileObj.createAxis('latitude', lats)
lat_axis.units = "degrees_north"
tme_axis = fileObj.createAxis('time', tmes)
tme_axis.units = "months"
var = fileObj.createVariable(
varname, numpy.float32,
(tme_axis, lat_axis, lon_axis)) # variable without data
var[:] = dataField # copy in the data
fileObj.close()
return None
def writeField(lons, lats, tmes, varname, dataField):
#-----------------------------------------------------------------------------------
#
# purpose: write an output field
#
# usage:
#
# passed : lons, lats, tmes, filename, varname, dataField
#
# returned: None
#
#-----------------------------------------------------------------------------------
fileObj = cdms2.createDataset(varname + '.nc')
lon_axis = fileObj.createAxis('longitude', lons)
lon_axis.units = "degrees_east"
lat_axis = fileObj.createAxis('latitude', lats)
lat_axis.units = "degrees_north"
tme_axis = fileObj.createAxis('time', tmes)
tme_axis.units = "months"
var = fileObj.createVariable(varname, numpy.float32, (tme_axis, lat_axis, lon_axis)) # variable without data
var[:] = dataField # copy in the data
fileObj.close()
return None
def saveGridFile(self, resultId, variable, createGridFile):
outdir = os.path.dirname(variable.gridfile)
outpath = os.path.join(outdir, resultId + ".nc")
if createGridFile:
try:
newDataset = cdms2.createDataset(outpath)
grid = variable.getGrid()
if not (grid is None):
axes = variable.getAxisList()
axis_keys = [axis.axis for axis in axes]
self.logger.info(" @RRR@ Copying axes: {0}".format(
str(axis_keys)))
for axis in axes:
if axis.isTime():
pass
else:
self.logger.info(
" @RRR@ axis: {0}, startVal: {1}".format(
axis.axis, axis[0]))
newDataset.copyAxis(axis)
newDataset.copyGrid(grid)
newDataset.close()
self.logger.info(
" @RRR@ saved Grid file: {0}".format(outpath))
except Exception as err:
self.logger.info(" @RRR@ Can't create grid file " + outpath +
": " + str(err))
return outpath
def write1D_4DField(varname, dataField, x, y=None, z=None, t=None):
#------------------------------------------------------------------------------
#
# purpose: write an output field which may be 1D, 2D, 3D or 4D to a NetCDF file
#
# usage: write1D_4DField(varname, dataField, x, y, z = None, t = None) for a 2D write
#
# passed : varname - name of the variable and the file id
# x,y,z,t - dimension vectors
# dataField - the data
#
# returned: None
#
#-------------------------------------------------------------------------------
import cdms2
fileObj = cdms2.createDataset(varname + '.nc')
# construct the axis tuple
x = x.astype(numpy.float64)
x_axis = fileObj.createAxis('x', x)
axisList = [x_axis]
if y is not None:
y = y.astype(numpy.float64)
y_axis = fileObj.createAxis('y', y)
axisList.append(y_axis)
if z is not None:
z = z.astype(numpy.float64)
z_axis = fileObj.createAxis('z', z)
axisList.append(z_axis)
if t is not None:
t = t.astype(numpy.float64)
t_axis = fileObj.createAxis('t', t)
axisList.append(t_axis)
if len(axisList) == 1:
axisTuple = (x_axis, )
else:
axisTuple = tuple(axisList)
# write the data to the file
var = fileObj.createVariable(varname, numpy.float32,
axisTuple) # variable without data
var[:] = dataField # copy in the data
fileObj.close()
return None
def write1D_4DField(varname, dataField, x, y = None, z = None, t = None):
#------------------------------------------------------------------------------
#
# purpose: write an output field which may be 1D, 2D, 3D or 4D to a NetCDF file
#
# usage: write1D_4DField(varname, dataField, x, y, z = None, t = None) for a 2D write
#
# passed : varname - name of the variable and the file id
# x,y,z,t - dimension vectors
# dataField - the data
#
# returned: None
#
#-------------------------------------------------------------------------------
import cdms2
fileObj = cdms2.createDataset(varname + '.nc')
# construct the axis tuple
x = x.astype(numpy.float64)
x_axis = fileObj.createAxis('x', x)
axisList = [x_axis]
if y is not None:
y = y.astype(numpy.float64)
y_axis = fileObj.createAxis('y', y)
axisList.append(y_axis)
if z is not None:
z = z.astype(numpy.float64)
z_axis = fileObj.createAxis('z', z)
axisList.append(z_axis)
if t is not None:
t = t.astype(numpy.float64)
t_axis = fileObj.createAxis('t', t)
axisList.append(t_axis)
if len(axisList) == 1:
axisTuple = (x_axis,)
else:
axisTuple = tuple(axisList)
# write the data to the file
var = fileObj.createVariable(varname, numpy.float32, axisTuple) # variable without data
var[:] = dataField # copy in the data
fileObj.close()
return None
def saveMetadata( self, resultId, variable ):
axes = variable.getAxisList()
grid = variable.getGrid()
outdir = os.path.dirname( variable.gridfile )
outpath = os.path.join(outdir, resultId + ".nc" )
newDataset = cdms2.createDataset( outpath )
for axis in axes: newDataset.copyAxis(axis)
newDataset.copyGrid(grid)
newDataset.close()
self.logger.info( "Saved metadata file: {0}".format( outpath ) )
return outpath
def main(var,
infile,
outfile,
time_bounds=':',
lon_bounds=':',
lat_bounds=':',
level_bounds=':'):
"""Run the program."""
cf = cdms2.open(infile)
if var == 'None':
vars = list_nobounds(cf, ids=True)
else:
vars = [var]
cfout = cdms2.createDataset(outfile)
nwritten = 0
lat_bounds = map(convert_lat,
lat_bounds) if lat_bounds != ':' else lat_bounds
lon_bounds = map(convert_lon,
lon_bounds) if lon_bounds != ':' else lon_bounds
if lon_bounds != ':':
assert lon_bounds[0] <= lon_bounds[1], \
"WEST_LON is not west of EAST_LON on a 0E - 360E interval"
for var in vars:
v = cf(var,
time=time_bounds,
longitude=lon_bounds,
latitude=lat_bounds,
level=level_bounds)
time_check = check_time_axis(v)
check_valid_range(v)
if time_check:
vout = cfout.write(v, axes=v.getAxisList(), id=v.id)
if hasattr(v, 'name') and 'variable' not in v.name:
vout.name = v.name
nwritten += 1
if nwritten == 0:
cfout.close()
os.remove(outfile)
sys.exit(1)
for att in cf.listglobal():
setattr(cfout, att, cf.attributes[att])
cfout.close()
def writeResults( self, pvar ):
operation_metadata = self.task_metadata[ 'operation' ]
output_name = operation_metadata.get( 'name', "TemporalProcessing" )
output_dir = operation_metadata.get( 'dir', os.path.dirname( pvar.dataset_path ) )
results = pvar.getLocalVariables()
output_path = os.path.join( output_dir, output_name )
for ( timestamp, result_var ) in results.items():
outfilename = "%s-%s.nc" % ( output_path, str(timestamp) )
outfile = cdms2.createDataset( outfilename )
outfile.write( result_var )
outfile.close()
time_axis = result_var.getTime()
print "Proc %d: Wrote %s slab to file %s" % ( pvar.global_comm.Get_rank(), str(result_var.shape), outfilename )
def saveGridFile( self, resultId, variable ):
grid = variable.getGrid()
outpath = None
if( grid != None ):
axes = variable.getAxisList()
outdir = os.path.dirname( variable.gridfile )
outpath = os.path.join(outdir, resultId + ".nc" )
newDataset = cdms2.createDataset( outpath )
for axis in axes:
if axis.isTime: pass
else: newDataset.copyAxis(axis)
newDataset.copyGrid(grid)
newDataset.close()
self.logger.info( "Saved grid file: {0}".format( outpath ) )
return outpath
def execute( self, args ):
( time_index, fname, varname, specs ) = args
default_outfile = 'wrfout'
t0 = specs.getFloat( 't0', 0.0 )
dt = specs.getFloat( 'dt', 1.0 )
time_data = [ t0 + dt*time_index ]
result_file = specs.getStr( 'outfile', os.path.expanduser( default_outfile ) )
time_units = specs.getStr( 'time_units', 'hours since 2000-01-01' )
data_location = specs.getStr( 'data_location', '~' )
output_dataset_directory = specs.getStr( 'output_dataset_directory', '.' )
fpath = os.path.join( data_location, fname )
try:
print " P[%d]: opening file: %s " % ( self.proc_index, fpath ); sys.stdout.flush()
cdms_file = cdms2.open( fpath )
except Exception:
print>>sys.stderr, "Can't read file %s " % ( fpath )
return
try:
print " P[%d]: opening var: %s. " % ( self.proc_index, varname ); sys.stdout.flush()
wrf_var = cdms_file( varname )
print " P[%d]: Done read. " % ( self.proc_index ); sys.stdout.flush()
except Exception:
print>>sys.stderr, "Variable %s does not seem to exist in file %s " % ( varname, fpath )
return
print " P[%d]: Regridding variable %s[%.2f] in file %s" % ( self.proc_index, varname, time_data[0], fpath ); sys.stdout.flush()
try:
time_axis = cdms2.createAxis( time_data )
time_axis.designateTime()
time_axis.id = "Time"
time_axis.units = time_units
var = standard_regrid( cdms_file, wrf_var, logfile='/tmp/regrid_log_%s_%d.txt' % (varname,time_index), cache = self.product_cache, iproc = self.proc_index )
axis_list = [ time_axis ]
axis_list.extend( var.getAxisList() )
var.coordinates = None
var.name = varname
outfile_path = os.path.join( output_dataset_directory, "%s-%s-%d.nc" % ( result_file, varname, time_index ) )
print " P[%d]: Writing to outfile %s" % ( self.proc_index, outfile_path ); sys.stdout.flush()
outfile = cdms2.createDataset( outfile_path )
outfile.write( var, extend=1, axes=axis_list, index=0 )
outfile.close()
except Exception, err:
print>>sys.stderr, " P[%d]: Error regridding data: %s " % ( self.proc_index, str(err ) ); sys.stderr.flush()
traceback.print_exc()
return 1
def main(var, infile, outfile,
time_bounds=':', lon_bounds=':',
lat_bounds=':', level_bounds=':'):
"""Run the program."""
cf = cdms2.open(infile)
if var == 'None':
vars = list_nobounds(cf, ids=True)
else:
vars = [var]
cfout = cdms2.createDataset(outfile)
nwritten = 0
lat_bounds = map(convert_lat, lat_bounds) if lat_bounds != ':' else lat_bounds
lon_bounds = map(convert_lon, lon_bounds) if lon_bounds != ':' else lon_bounds
if lon_bounds != ':':
assert lon_bounds[0] <= lon_bounds[1], \
"WEST_LON is not west of EAST_LON on a 0E - 360E interval"
for var in vars:
v = cf(var, time=time_bounds, longitude=lon_bounds,
latitude=lat_bounds, level=level_bounds)
time_check = check_time_axis(v)
check_valid_range(v)
if time_check:
vout = cfout.write(v, axes=v.getAxisList(), id=v.id)
if hasattr(v, 'name') and 'variable' not in v.name:
vout.name = v.name
nwritten += 1
if nwritten == 0:
cfout.close()
os.remove(outfile)
sys.exit(1)
for att in cf.listglobal():
setattr(cfout, att, cf.attributes[att])
cfout.close()
#!/usr/bin/env python
import cdms2, numpy, os, sys
from markError import NTIME,NLAT,NLON,x,clearError,markError,reportError
from markError import get_sample_data_dir
clearError()
print 'Test 4: CdmsFile [numpy.ma] read/write ...',
time = numpy.ma.array([0.0,366.0,731.0])
lat = numpy.ma.arange(NLAT)*(180./(NLAT-1))-90.
lon = numpy.ma.arange(NLON)*(360.0/NLON)
timestr = ['2000','2001','2002']
u = x[0]
f = cdms2.createDataset('readwrite.nc')
h = cdms2.open(os.path.join(get_sample_data_dir(),'readonly.nc'))
tobj = f.createAxis('time',numpy.ma.array([time[1]]))
tobj.units = 'days since 2000-1-1'
latobj = f.createAxis('latitude',lat)
latobj.units = 'degrees_north'
lonobj = f.createAxis('longitude',lon)
lonobj.units = 'degrees_east'
var = f.createVariable('u',numpy.float,(tobj,latobj,lonobj))
var.units = 'm/s'
try:
var[:]=u[0]
except:
markError("Setting a slice")
try:
var.assignValue(u[0])
def process_file(ifile,suffix,average=False,forcedaily=False,mask=True,xlist=[]):
try:
d = cdms2.open(ifile)
except:
print "Error opening file", ifile
usage()
sys.exit(1)
hcrit = 0.5 # Critical value of Heavyside function for inclusion.
ofilelist = []
for vn in d.variables:
var = d.variables[vn]
# Need to check whether it really has a stash_item to skip coordinate variables
# Note: need to match both item and section number
if not hasattr(var,'stash_item'):
continue
item_code = var.stash_section[0]*1000 + var.stash_item[0]
if item_code in xlist:
print "Skipping", item_code
continue
grid = var.getGrid()
time = var.getTime()
timevals = np.array(time[:])
if forcedaily:
# Work around cdms error in times
for k in range(len(time)):
timevals[k] = round(timevals[k],1)
umvar = stashvar.StashVar(item_code,var.stash_model[0])
vname = umvar.name
print vname, var[0,0,0,0]
# Create filename from variable name and cell_methods,
# checking for name collisions
if suffix:
ofile = "%s_%s.nc" % (umvar.uniquename, suffix)
else:
ofile = "%s.nc" % umvar.uniquename
if ofile in ofilelist:
raise Exception("Duplicate file name %s" % ofile)
ofilelist.append(ofile)
# If output file exists then append to it, otherwise create a new file
try:
file = cdms2.openDataset(ofile, 'r+')
newv = file.variables[vname]
newtime = newv.getTime()
except cdms2.error.CDMSError:
file = cdms2.createDataset(ofile)
# Stop it creating the bounds_latitude, bounds_longitude variables
cdms2.setAutoBounds("off")
# By default get names like latitude0, longitude1
# Need this awkwardness to get the variable/dimension name set correctly
# Is there a way to change the name cdms uses after
# newlat = newgrid.getLatitude() ????
newlat = file.createAxis('lat', grid.getLatitude()[:])
newlat.standard_name = "latitude"
newlat.axis = "Y"
newlat.units = 'degrees_north'
newlon = file.createAxis('lon', grid.getLongitude()[:])
newlon.standard_name = "longitude"
newlon.axis = "X"
newlon.units = 'degrees_east'
order = var.getOrder()
if order[1] == 'z':
lev = var.getLevel()
if len(lev) > 1:
newlev = file.createAxis('lev', lev[:])
for attr in ('standard_name', 'units', 'positive', 'axis'):
if hasattr(lev,attr):
setattr(newlev, attr, getattr(lev,attr))
else:
newlev = None
else:
# Pseudo-dimension
pdim = var.getAxis(1)
if len(pdim) > 1:
newlev = file.createAxis('pseudo', pdim[:])
else:
newlev = None
newtime = file.createAxis('time', None, cdms2.Unlimited)
newtime.standard_name = "time"
newtime.units = time.units # "days since " + `baseyear` + "-01-01 00:00"
newtime.setCalendar(time.getCalendar())
newtime.axis = "T"
if var.dtype == np.dtype('int32'):
vtype = cdms2.CdInt
missval = -2147483647
else:
vtype = cdms2.CdFloat
missval = 1.e20
if newlev:
newv = file.createVariable(vname, vtype, (newtime, newlev, newlat, newlon))
else:
newv = file.createVariable(vname, vtype, (newtime, newlat, newlon))
for attr in ("standard_name", "long_name", "units"):
if hasattr(umvar, attr):
newv.setattribute(attr, getattr(umvar,attr))
newv.missing_value = missval
newv.stash_section=var.stash_section[0]
newv.stash_item=var.stash_item[0]
newv._FillValue = missval
try:
newv.units = var.units
except AttributeError:
pass
# Get appropriate file position
# Uses 360 day calendar, all with same base time so must be 30 days on.
k = len(newtime)
# float needed here to get the later logical tests to work properly
avetime = float(MV.average(timevals[:])) # Works in either case
if k>0:
if average:
# if newtime[-1] != (avetime - 30):
# For Gregorian calendar relax this a bit
# Sometimes get differences slightly > 31
if not 28 <= avetime - newtime[-1] <= 31.5:
raise error, "Times not consecutive %f %f %f" % (newtime[-1], avetime, timevals[0])
else:
if k > 1:
# Need a better test that works when k = 1. This is just a
# temporary workaround
if not np.allclose( newtime[-1] + (newtime[-1]-newtime[-2]), timevals[0] ):
raise error, "Times not consecutive %f %f " % (newtime[-1], timevals[0])
if (30201 <= item_code <= 30303) and mask:
# P LEV/UV GRID with missing values treated as zero.
# Needs to be corrected by Heavyside fn
heavyside = d.variables['psag']
# Check variable code as well as the name.
if heavyside.stash_item[0] != 301 or heavyside.stash_section[0] != 30:
raise error, "Heavyside variable code mismatch"
if average:
newtime[k] = avetime
if var.shape[1] > 1:
# multiple levels
newv[k] = MV.average(var[:],axis=0).astype(np.float32)
else:
# single level
newv[k] = MV.average(var[:],axis=0)[0].astype(np.float32)
else:
for i in range(len(timevals)):
if var.shape[1] > 1:
# Multi-level
if (30201 <= item_code <= 30303) and mask:
newv[k+i] = np.where( np.greater(heavyside[i], hcrit), var[i]/heavyside[0], newv.getMissing())
else:
newv[k+i] = var[i]
else:
newv[k+i] = var[i,0]
newtime[k+i] = timevals[i]
file.close()
hcrit = 0.5 # Critical value of Heavyside function for inclusion.
# print "LEN(TIME)", len(time)
# If output file exists then append to it, otherwise create a new file
try:
file = cdms2.openDataset(ofile, 'r+')
newv = file.variables[vname]
newtime = newv.getTime()
except cdms2.error.CDMSError:
if not usenc4:
# Force netCDF3 output
cdms2.setNetcdfShuffleFlag(0)
cdms2.setNetcdfDeflateFlag(0)
cdms2.setNetcdfDeflateLevelFlag(0)
file = cdms2.createDataset(ofile)
file.history = "Created by um2netcdf.py."
# Stop it creating the bounds_latitude, bounds_longitude variables
cdms2.setAutoBounds("off")
# By default get names like latitude0, longitude1
# Need this awkwardness to get the variable/dimension name set correctly
# Is there a way to change the name cdms uses after
# newlat = newgrid.getLatitude() ????
newlat = file.createAxis('lat', grid.getLatitude()[:])
newlat.standard_name = "latitude"
newlat.axis = "Y"
newlat.units = 'degrees_north'
newlon = file.createAxis('lon', grid.getLongitude()[:])
newlon.standard_name = "longitude"
newlon.axis = "X"
def main(var, incat, output,
force,start_year=None,end_year=None):
"""Run the program.
@param var: The variable to extract from the catalouge.
@type var: string
@param input: single or list of input files
@type input: list or string
@param output: File to output list of actual netCDF files to
@type output: string
@param force: If True then existing files will be overwritten. If false then
they will be skipped.
@type force: boolean
"""
#
# Get subset from catalogues
#
if not force and os.access(output, os.F_OK):
return
if start_year and end_year:
syear,smonth,sday = split_date(start_year)
eyear,emonth,eday = split_date(end_year,False)
start_year = cdtime.comptime(syear,smonth,sday)
end_year = cdtime.comptime(eyear,emonth,eday)
cf = cdms2.open(incat)
if var == 'None':
from cct.cct import list_nobounds
vars = list_nobounds(cf, ids=True)
else:
vars = [var]
recall_files(cf, vars[0])
cfout = cdms2.createDataset(output)
nwritten = 0
for var in vars:
try:
if start_year and end_year:
#Check time axis
v = cf(var,time=(start_year,end_year,'cc'))
t = v.getTime()
v_start_year = cdtime.reltime(t[0],t.units).tocomp(t.getCalendar())
v_end_year = cdtime.reltime(t[-1],t.units).tocomp(t.getCalendar())
if not (v_start_year.year == start_year.year and \
v_start_year.month == start_year.month and \
v_end_year.year == end_year.year and \
v_end_year.month == end_year.month):
raise Exception("Time axis does not contain requested period, Request (%s - %s), Actual (%s - %s)" %
(start_year.year,end_year.year,v_start_year.year,v_end_year.year))
else:
v = cf(var)
current_time = v.getTime()
if current_time is None: raise Exception('Not time axis.')
except Exception, e:
# Probably no data in file...
print 'WARNING: skipping variable, %s, in file ' % var, incat
print str(e)
continue
# Check that the time-axis is monotonically increasing...
# We need to discover the time gap between adjacent points
# then test that it lies within the correct range.
time_axis = v.getTime()
# Make a test difference between two adjacent time steps.
testdiff = time_axis[2] - time_axis[1]
if testdiff == 1:
# daily
tol = (1, 1)
elif 28 <= testdiff <= 31:
# monthly
tol = (28, 31)
elif 360 <= testdiff <= 366:
# yearly
tol = (360, 366)
diffs = time_axis[1:] - time_axis[:-1]
# check if any differences are outside bounds
if any(d < tol[0] or d > tol[1] for d in diffs):
print 'WARNING: incomplete time axis. Skipping ', var
cfout.close()
os.remove(output)
sys.exit(0)
if 'valid_range' in v.attributes and isinstance(v.valid_range,
basestring):
try:
v.valid_range = np.fromstring(v.valid_range.strip('[]'),
dtype=v.dtype,
sep=' ')
except:
pass
# Check if any axes are of type float32 instead of float64
# This is due to a known bug in cdat < 6.0
# TODO: Update to a newer version of cdat or learn how to construct
# non-rectangular grids with cdms2.
# Try to deal with situations were variables in the file do not have
# latitude or longitude axis.
try:
if v.getLatitude().dtype == 'float32' or v.getLongitude().dtype == 'float32':
# This is a hack to get around the bug in cdat < 6.0 that stops netcdf
# files being written out if their lat or lon are floats rather
# than doubles.
print("### WARNING - File has float coordinate variables instead of doubles ###")
print(" Making multiple attempts to write out file")
attempts = 0
done = False
while attempts < 4 and not done:
try:
vout = cfout.write(v, id = v.id, axes = v.getAxisList())
print("Written!")
done = True
except TypeError:
attempts += 1
print("Failed attempts = " + str(attempts))
continue
else:
vout = cfout.write(v, axes = v.getAxisList(), id=v.id)
# Catching non-lat/lon variables.
except AttributeError:
vout = cfout.write(v, axes = v.getAxisList(), id=v.id)
if hasattr(v, 'name') and 'variable' not in v.name:
vout.name = v.name
nwritten += 1
def execute(self, args):
(time_index, fname, varname, specs) = args
default_outfile = 'wrfout'
t0 = specs.getFloat('t0', 0.0)
dt = specs.getFloat('dt', 1.0)
time_data = [t0 + dt * time_index]
result_file = specs.getStr('outfile',
os.path.expanduser(default_outfile))
time_units = specs.getStr('time_units', 'hours since 2000-01-01')
data_location = specs.getStr('data_location', '~')
output_dataset_directory = specs.getStr('output_dataset_directory',
'.')
fpath = os.path.join(data_location, fname)
try:
print " P[%d]: opening file: %s " % (self.proc_index, fpath)
sys.stdout.flush()
cdms_file = cdms2.open(fpath)
except Exception:
print >> sys.stderr, "Can't read file %s " % (fpath)
return
try:
print " P[%d]: opening var: %s. " % (self.proc_index, varname)
sys.stdout.flush()
wrf_var = cdms_file(varname)
print " P[%d]: Done read. " % (self.proc_index)
sys.stdout.flush()
except Exception:
print >> sys.stderr, "Variable %s does not seem to exist in file %s " % (
varname, fpath)
return
print " P[%d]: Regridding variable %s[%.2f] in file %s" % (
self.proc_index, varname, time_data[0], fpath)
sys.stdout.flush()
try:
time_axis = cdms2.createAxis(time_data)
time_axis.designateTime()
time_axis.id = "Time"
time_axis.units = time_units
var = standard_regrid(cdms_file,
wrf_var,
logfile='/tmp/regrid_log_%s_%d.txt' %
(varname, time_index),
cache=self.product_cache,
iproc=self.proc_index)
axis_list = [time_axis]
axis_list.extend(var.getAxisList())
var.coordinates = None
var.name = varname
outfile_path = os.path.join(
output_dataset_directory,
"%s-%s-%d.nc" % (result_file, varname, time_index))
print " P[%d]: Writing to outfile %s" % (self.proc_index,
outfile_path)
sys.stdout.flush()
outfile = cdms2.createDataset(outfile_path)
outfile.write(var, extend=1, axes=axis_list, index=0)
outfile.close()
except Exception, err:
print >> sys.stderr, " P[%d]: Error regridding data: %s " % (
self.proc_index, str(err))
sys.stderr.flush()
traceback.print_exc()
return 1
import cdms2
inputPath = "http://aims3.llnl.gov/thredds/dodsC/cmip5_css02_data/cmip5/output1/CMCC/CMCC-CESM/historical/mon/atmos/Amon/r1i1p1/tas/1/tas_Amon_CMCC-CESM_historical_r1i1p1_198001-198412.nc"
dataset = cdms2.open(inputPath)
resolution = 128
regridder = "regrid2"
input = dataset("tas")
t42 = cdms2.createGaussianGrid(resolution)
result = input.regrid(t42, regridTool=regridder)
axes = result.getAxisList()
grid = result.getGrid()
newDataset = cdms2.createDataset("/tmp/test")
for axis in axes:
newDataset.copyAxis(axis)
newDataset.copyGrid(grid)
newDataset.createVariableCopy(result)
newDataset.close()
print "."
def execute(self, args):
(time_index, fname, varname, specs) = args
default_outfile = 'cam_cs'
result_file = specs.getStr('outfile',
os.path.expanduser(default_outfile))
time_units = specs.getStr('time_units', 'hours since 2000-01-01')
data_location = specs.getStr('data_location', '~')
output_dataset_directory = specs.getStr('output_dataset_directory',
'.')
fpath = os.path.join(data_location, fname)
try:
print " P[%d]: opening file: %s " % (self.proc_index, fpath)
sys.stdout.flush()
cdms_file = cdms2.open(fpath)
except Exception:
print >> sys.stderr, "Can't read file %s " % (fpath)
return
try:
print " P[%d]: opening var: %s. " % (self.proc_index, varname)
sys.stdout.flush()
var = cdms_file(varname)
print " P[%d]: Done read. " % (self.proc_index)
sys.stdout.flush()
except Exception:
print >> sys.stderr, "Variable %s does not seem to exist in file %s " % (
varname, fpath)
return
nTiles = 6
iTile = 3
ndims = len(var.shape)
if ndims == 3:
npts_tot = var.shape[2]
nlev = var.shape[1]
else:
npts_tot = var.shape[1]
nlev = 1
npts_tile = npts_tot / nTiles
dim = int(math.sqrt(npts_tile))
iOffset = 4 * dim
npts_tile = dim * dim
iStart = 1 + iOffset + npts_tile * iTile
iEnd = iStart + npts_tile
time_axis = var.getAxisList('time')
x_axis = cdms2.createAxis(range(dim))
x_axis.id = 'x'
x_axis.axis = 'X'
y_axis = cdms2.createAxis(range(dim))
y_axis.id = 'y'
y_axis.axis = 'Y'
axes = [y_axis, x_axis]
if ndims == 3:
var_data = var[0, 0:nlev, iStart:iEnd]
levaxis = var.getLevel()
if levaxis == None:
levaxis = cdms2.createAxis(range(nlev))
y_axis.id = 'z'
y_axis.axis = 'Z'
axes.insert(0, levaxis)
new_shape = [nlev, dim, dim]
else:
var_data = var[0, iStart:iEnd]
new_shape = [dim, dim]
if len(time_axis):
axes.insert(0, time_axis[0])
new_shape.insert(0, 1)
var_data = var_data.reshape(new_shape)
tVar = cdms2.createVariable(var_data,
axes=axes,
id=var.id,
typecode=var.typecode(),
order='C')
outfile_path = os.path.join(
output_dataset_directory,
"%s-%s-%d.nc" % (result_file, varname, time_index))
print " P[%d]: Writing to outfile %s" % (self.proc_index, outfile_path)
sys.stdout.flush()
outfile = cdms2.createDataset(outfile_path)
outfile.write(tVar)
outfile.close()
#!/usr/bin/env python
import cdms2, numpy, os, sys
from markError import NTIME, NLAT, NLON, x, clearError, markError, reportError
clearError()
print "Test 4: CdmsFile [numpy.ma] read/write ...",
time = numpy.ma.array([0.0, 366.0, 731.0])
lat = numpy.ma.arange(NLAT) * (180.0 / (NLAT - 1)) - 90.0
lon = numpy.ma.arange(NLON) * (360.0 / NLON)
timestr = ["2000", "2001", "2002"]
u = x[0]
f = cdms2.createDataset("readwrite4.nc")
pth = os.path.dirname(os.path.abspath(__file__))
h = cdms2.open(os.path.join(pth, "readonly.nc"))
tobj = f.createAxis("time", numpy.ma.array([time[1]]))
tobj.units = "days since 2000-1-1"
latobj = f.createAxis("latitude", lat)
latobj.units = "degrees_north"
lonobj = f.createAxis("longitude", lon)
lonobj.units = "degrees_east"
var = f.createVariable("u", numpy.float, (tobj, latobj, lonobj))
var.units = "m/s"
try:
var[:] = u[0]
except:
markError("Setting a slice")
try:
self._spec = spec
self._mode = mode
@abstractmethod
def execute(self): pass
if __name__ == "__main__":
metadata = { "axes": "13", "index": 0 }
print( str(metadata) )
if __name__ == "__main__":
from cdms2 import timeslice
dsetUri = "http://esgf.nccs.nasa.gov/thredds/dodsC/CMIP5/NASA/GISS/historical/E2-H_historical_r1i1p1/tas_Amon_GISS-E2-H_historical_r1i1p1_195101-200512.nc"
resolution = 128
dset = cdms2.open(dsetUri)
variable = dset["tas"](timeslice(0,1))
ingrid = variable.getGrid()
axes = variable.getAxisList()
grid = variable.getGrid()
outdir = os.path.dirname( variable.gridfile )
outpath = os.path.expanduser('~/.cdas/debug_grid_file.nc' )
newDataset = cdms2.createDataset( outpath )
for axis in axes: newDataset.copyAxis(axis)
newDataset.copyGrid(grid)
# newDataset.close()
NTIME = 3
NLAT = 16
NLON = 32
x = Numeric.arange(float(2*NTIME*NLAT*NLON))
x.shape=(2,NTIME,NLAT,NLON)
u = x[0]
v = x[1]
time = Numeric.array([0.0,366.0,731.0])
lat = Numeric.arange(NLAT)*(180./(NLAT-1))-90.
lon = Numeric.arange(NLON)*(360.0/NLON)
timestr = ['2000','2001','2002']
for id in ['u','v']:
for i in range(len(timestr)):
f = cdms.createDataset('%s_%s.nc'%(id,timestr[i]))
tobj = f.createAxis('time',Numeric.array([time[i]]))
tobj.units = 'days since 2000-1-1'
latobj = f.createAxis('latitude',lat)
latobj.units = 'degrees_north'
lonobj = f.createAxis('longitude',lon)
lonobj.units = 'degrees_east'
var = f.createVariable(id,cdms.CdDouble,(tobj,latobj,lonobj))
var.units = 'm/s'
var.missing_value = -99.9
if id=='u':
var[:]=u[i]
else:
var[:]=v[i]
f.Conventions = "CF-1.0"
NTIME = 3
NLAT = 16
NLON = 32
x = Numeric.arange(float(2 * NTIME * NLAT * NLON))
x.shape = (2, NTIME, NLAT, NLON)
u = x[0]
v = x[1]
time = Numeric.array([0.0, 366.0, 731.0])
lat = Numeric.arange(NLAT) * (180. / (NLAT - 1)) - 90.
lon = Numeric.arange(NLON) * (360.0 / NLON)
timestr = ['2000', '2001', '2002']
for id in ['u', 'v']:
for i in range(len(timestr)):
f = cdms.createDataset('%s_%s.nc' % (id, timestr[i]))
tobj = f.createAxis('time', Numeric.array([time[i]]))
tobj.units = 'days since 2000-1-1'
latobj = f.createAxis('latitude', lat)
latobj.units = 'degrees_north'
lonobj = f.createAxis('longitude', lon)
lonobj.units = 'degrees_east'
var = f.createVariable(id, cdms.CdDouble, (tobj, latobj, lonobj))
var.units = 'm/s'
var.missing_value = -99.9
if id == 'u':
var[:] = u[i]
else:
var[:] = v[i]
f.Conventions = "CF-1.0"
def execute( self, args ):
( time_index, fname, varname, specs ) = args
default_outfile = 'cam_cs'
result_file = specs.getStr( 'outfile', os.path.expanduser( default_outfile ) )
time_units = specs.getStr( 'time_units', 'hours since 2000-01-01' )
data_location = specs.getStr( 'data_location', '~' )
output_dataset_directory = specs.getStr( 'output_dataset_directory', '.' )
fpath = os.path.join( data_location, fname )
try:
print " P[%d]: opening file: %s " % ( self.proc_index, fpath ); sys.stdout.flush()
cdms_file = cdms2.open( fpath )
except Exception:
print>>sys.stderr, "Can't read file %s " % ( fpath )
return
try:
print " P[%d]: opening var: %s. " % ( self.proc_index, varname ); sys.stdout.flush()
var = cdms_file( varname )
print " P[%d]: Done read. " % ( self.proc_index ); sys.stdout.flush()
except Exception:
print>>sys.stderr, "Variable %s does not seem to exist in file %s " % ( varname, fpath )
return
nTiles = 6
iTile = 3
ndims = len( var.shape )
if ndims == 3:
npts_tot = var.shape[2]
nlev = var.shape[1]
else:
npts_tot = var.shape[1]
nlev = 1
npts_tile = npts_tot / nTiles
dim = int( math.sqrt( npts_tile ) )
iOffset = 4*dim
npts_tile = dim*dim
iStart = 1 + iOffset + npts_tile * iTile
iEnd = iStart + npts_tile
time_axis = var.getAxisList('time')
x_axis = cdms2.createAxis( range(dim) )
x_axis.id = 'x'
x_axis.axis = 'X'
y_axis = cdms2.createAxis( range(dim) )
y_axis.id = 'y'
y_axis.axis = 'Y'
axes =[ y_axis, x_axis ]
if ndims == 3:
var_data = var[ 0, 0:nlev, iStart : iEnd ]
levaxis = var.getLevel()
if levaxis == None:
levaxis = cdms2.createAxis( range(nlev) )
y_axis.id = 'z'
y_axis.axis = 'Z'
axes.insert( 0, levaxis )
new_shape = [ nlev, dim, dim ]
else:
var_data = var[ 0, iStart : iEnd ]
new_shape = [ dim, dim ]
if len( time_axis ):
axes.insert( 0, time_axis[0] )
new_shape.insert( 0, 1 )
var_data = var_data.reshape( new_shape )
tVar = cdms2.createVariable( var_data, axes=axes, id=var.id, typecode=var.typecode(), order='C' )
outfile_path = os.path.join( output_dataset_directory, "%s-%s-%d.nc" % ( result_file, varname, time_index ) )
print " P[%d]: Writing to outfile %s" % ( self.proc_index, outfile_path ); sys.stdout.flush()
outfile = cdms2.createDataset( outfile_path )
outfile.write( tVar )
outfile.close()
