def shiftGrid(infile, outfile, variable, latShift=0, lonShift=-280):
if os.path.exists(outfile): os.remove(outfile)
thisfile=cdms2.open(infile)
thisLon = thisfile[variable].getLongitude()
thisLon[:].data[:] = (thisLon[:].data[:] - lonShift)%360
newvar = cdms2.createVariable(MV.array(thisfile[variable][:]), id=variable, fill_value=1.e20)
newvar.setAxisList((thisfile[variable].getTime(), thisfile[variable].getLatitude(), thisLon))
if os.path.exists(outfile): os.remove(outfile)
outFile = cdms2.open(outfile,'w')
outFile.write(newvar)
thisfile.close()
outFile.close()
sys.exit(1)
#------------------------------------------------------------------------------------------------
# Create canvas ---
canvas = vcs.init(geometry=(1200,800))
canvas.open()
template = canvas.createtemplate()
template.blank(['title','mean','min','max','dataname','crdate','crtime','units']) ## Turn off additional information
#template.list() ## This commend could give list of items
# Set ploting range ---
lat1=-60
lat2=80
lon1=0
lon2=360
# Wind speed ---
data['wnd'] = MV.sqrt(data['ua']**2+data['va']**2)
iso = canvas.createisofill()
iso.datawc_x1 = lon1
iso.datawc_x2 = lon2
iso.datawc_y1 = lat1
iso.datawc_y2 = lat2
canvas.setcolormap('blue_to_orange')
canvas.plot(data['wnd'],iso,template)
# Geopotential height field ---
lines1 = vcs.createisoline()
lines1.datawc_x1 = lon1
lines1.datawc_x2 = lon2
lines1.datawc_y1 = lat1
lines1.datawc_y2 = lat2
lines1.linecolors = ['black']
def process_file(ifile,suffix,average=False,forcedaily=False,mask=True,xlist=[]):
try:
d = cdms2.open(ifile)
except:
print "Error opening file", ifile
usage()
sys.exit(1)
hcrit = 0.5 # Critical value of Heavyside function for inclusion.
ofilelist = []
for vn in d.variables:
var = d.variables[vn]
# Need to check whether it really has a stash_item to skip coordinate variables
# Note: need to match both item and section number
if not hasattr(var,'stash_item'):
continue
item_code = var.stash_section[0]*1000 + var.stash_item[0]
if item_code in xlist:
print "Skipping", item_code
continue
grid = var.getGrid()
time = var.getTime()
timevals = np.array(time[:])
if forcedaily:
# Work around cdms error in times
for k in range(len(time)):
timevals[k] = round(timevals[k],1)
umvar = stashvar.StashVar(item_code,var.stash_model[0])
vname = umvar.name
print vname, var[0,0,0,0]
# Create filename from variable name and cell_methods,
# checking for name collisions
if suffix:
ofile = "%s_%s.nc" % (umvar.uniquename, suffix)
else:
ofile = "%s.nc" % umvar.uniquename
if ofile in ofilelist:
raise Exception("Duplicate file name %s" % ofile)
ofilelist.append(ofile)
# If output file exists then append to it, otherwise create a new file
try:
file = cdms2.openDataset(ofile, 'r+')
newv = file.variables[vname]
newtime = newv.getTime()
except cdms2.error.CDMSError:
file = cdms2.createDataset(ofile)
# Stop it creating the bounds_latitude, bounds_longitude variables
cdms2.setAutoBounds("off")
# By default get names like latitude0, longitude1
# Need this awkwardness to get the variable/dimension name set correctly
# Is there a way to change the name cdms uses after
# newlat = newgrid.getLatitude() ????
newlat = file.createAxis('lat', grid.getLatitude()[:])
newlat.standard_name = "latitude"
newlat.axis = "Y"
newlat.units = 'degrees_north'
newlon = file.createAxis('lon', grid.getLongitude()[:])
newlon.standard_name = "longitude"
newlon.axis = "X"
newlon.units = 'degrees_east'
order = var.getOrder()
if order[1] == 'z':
lev = var.getLevel()
if len(lev) > 1:
newlev = file.createAxis('lev', lev[:])
for attr in ('standard_name', 'units', 'positive', 'axis'):
if hasattr(lev,attr):
setattr(newlev, attr, getattr(lev,attr))
else:
newlev = None
else:
# Pseudo-dimension
pdim = var.getAxis(1)
if len(pdim) > 1:
newlev = file.createAxis('pseudo', pdim[:])
else:
newlev = None
newtime = file.createAxis('time', None, cdms2.Unlimited)
newtime.standard_name = "time"
newtime.units = time.units # "days since " + `baseyear` + "-01-01 00:00"
newtime.setCalendar(time.getCalendar())
newtime.axis = "T"
if var.dtype == np.dtype('int32'):
vtype = cdms2.CdInt
missval = -2147483647
else:
vtype = cdms2.CdFloat
missval = 1.e20
if newlev:
newv = file.createVariable(vname, vtype, (newtime, newlev, newlat, newlon))
else:
newv = file.createVariable(vname, vtype, (newtime, newlat, newlon))
for attr in ("standard_name", "long_name", "units"):
if hasattr(umvar, attr):
newv.setattribute(attr, getattr(umvar,attr))
newv.missing_value = missval
newv.stash_section=var.stash_section[0]
newv.stash_item=var.stash_item[0]
newv._FillValue = missval
try:
newv.units = var.units
except AttributeError:
pass
# Get appropriate file position
# Uses 360 day calendar, all with same base time so must be 30 days on.
k = len(newtime)
# float needed here to get the later logical tests to work properly
avetime = float(MV.average(timevals[:])) # Works in either case
if k>0:
if average:
# if newtime[-1] != (avetime - 30):
# For Gregorian calendar relax this a bit
# Sometimes get differences slightly > 31
if not 28 <= avetime - newtime[-1] <= 31.5:
raise error, "Times not consecutive %f %f %f" % (newtime[-1], avetime, timevals[0])
else:
if k > 1:
# Need a better test that works when k = 1. This is just a
# temporary workaround
if not np.allclose( newtime[-1] + (newtime[-1]-newtime[-2]), timevals[0] ):
raise error, "Times not consecutive %f %f " % (newtime[-1], timevals[0])
if (30201 <= item_code <= 30303) and mask:
# P LEV/UV GRID with missing values treated as zero.
# Needs to be corrected by Heavyside fn
heavyside = d.variables['psag']
# Check variable code as well as the name.
if heavyside.stash_item[0] != 301 or heavyside.stash_section[0] != 30:
raise error, "Heavyside variable code mismatch"
if average:
newtime[k] = avetime
if var.shape[1] > 1:
# multiple levels
newv[k] = MV.average(var[:],axis=0).astype(np.float32)
else:
# single level
newv[k] = MV.average(var[:],axis=0)[0].astype(np.float32)
else:
for i in range(len(timevals)):
if var.shape[1] > 1:
# Multi-level
if (30201 <= item_code <= 30303) and mask:
newv[k+i] = np.where( np.greater(heavyside[i], hcrit), var[i]/heavyside[0], newv.getMissing())
else:
newv[k+i] = var[i]
else:
newv[k+i] = var[i,0]
newtime[k+i] = timevals[i]
file.close()
canvas = vcs.init(geometry=(1200, 800))
canvas.open()
template = canvas.createtemplate()
template.blank(
['title', 'mean', 'min', 'max', 'dataname', 'crdate', 'crtime',
'units']) ## Turn off additional information
#template.list() ## This commend could give list of items
# Set ploting range ---
lat1 = -60
lat2 = 80
lon1 = 0
lon2 = 360
# Wind speed ---
data['wnd'] = MV.sqrt(data['ua']**2 + data['va']**2)
iso = canvas.createisofill()
iso.datawc_x1 = lon1
iso.datawc_x2 = lon2
iso.datawc_y1 = lat1
iso.datawc_y2 = lat2
canvas.setcolormap('blue_to_orange')
canvas.plot(data['wnd'], iso, template)
# Geopotential height field ---
lines1 = vcs.createisoline()
lines1.datawc_x1 = lon1
lines1.datawc_x2 = lon2
lines1.datawc_y1 = lat1
lines1.datawc_y2 = lat2
lines1.linecolors = ['black']
def do_yearlyWPAvg(sstdir, sstrootname, variable, outdir, yearStart, yearEnd, threshold=28.5 + 273.15, latWindow=None):
nodata = 1.0e20
varUnits = None
thisFilter = numpy.ones((3, 3))
# thisFilter[1][1]=1
EarthSurface = 510072000
factor = 1 # (2*85*360) /( 360.0*180.0)
areaWP = []
latWindowMatrix = None
for iyear in range(yearStart, yearEnd + 1):
tempAvg = None
wdata = None
weights = None
dimVar = None
counter = None
print "Processing year {0}".format(iyear)
for imonth in range(1, 12 + 1):
idate = "{0}{1:02}".format(iyear, imonth)
fname = "{0}/{1}_{2}.nc".format(sstdir, sstrootname, idate)
thisFile = cdms2.open(fname)
thisVar = numpy.ravel(thisFile[variable][:])
if latWindow is not None:
if latWindowMatrix is None:
latWindowMatrix = numpy.zeros(thisFile[variable].shape)
for ii in xrange(latWindow[0], latWindow[1] + 1):
latWindowMatrix[:, ii] = 1
thisVarTmp = thisVar
thisVar = numpy.multiply(thisVarTmp, numpy.ravel(latWindowMatrix))
if tempAvg is None: # settings
thisGrid = thisFile[variable].getGrid()
if thisGrid is None:
thisGrid = makeGrid()
(latws, lonws) = thisGrid.getWeights()
weights = MV.outerproduct(latws, lonws)
wdata = thisVar < nodata
tempAvg = numpy.zeros(thisVar.shape)
tempAvg[wdata] = thisVar[wdata]
dimVar = numpy.squeeze(thisFile[variable][:]).shape
counter = numpy.zeros(tempAvg.shape, dtype="float")
counter[wdata] = 1
else:
wdata = thisVar < nodata
counter[wdata] = counter[wdata] + 1
tempAvg[wdata] = tempAvg[wdata] + thisVar[wdata]
# compute average
wdivide = counter > 0
avg = numpy.zeros(tempAvg.shape)
if wdivide.any:
avg[wdivide] = tempAvg[wdivide] / counter[wdivide]
# set to areas < threshold
wtzero = avg < threshold
avg[wtzero] = 0
# compute current area
warea = (avg >= threshold) * (avg < nodata)
area = factor * EarthSurface * numpy.ravel(weights)[warea].sum()
areaWP.append([iyear, area])
# create variables
outAreaTmp = numpy.reshape(avg, dimVar)
# filter before saving: grid stiching area: less data for ensemble mean here
outAreaBis = do_interp(outAreaTmp, threshold, nodata, 156, 156 + 3, 0, 4 * 80) # rebuild
outArea = do_convolve(outAreaBis, thisFilter, threshold, nodata, 156 + 2, 156 + 5, 0, 4 * 80) # smoothen
wpOut = cdms2.createVariable(
outArea,
typecode="f",
id="warmpool",
grid=thisGrid,
copyaxes=1,
attributes=dict(long_name="warmpool, average temperature method, year {0}".format(iyear), units=varUnits),
)
# write to file
outfilename = "{0}/warmpool_{1}.nc".format(outdir, iyear)
if os.path.exists(outfilename):
os.remove(outfilename)
outfile = cdms2.open(outfilename, "w")
outfile.write(wpOut)
outfile.close()
# close files
thisFile.close()
return areaWP
def do_yearlyWPall(sstdir, sstrootname, variable, outdir, yearStart, yearEnd, latWindow=None):
threshold = 28 + 273.15
nodata = 1.0e20
varUnits = None
referenceGrid = None
referenceGrid = makeGrid()
latws, lonwts = referenceGrid.getWeights()
weights = MV.outerproduct(latws, lonwts)
EarthSurface = 510072000
areaWP = []
latWindowMatrix = None
for iyear in range(yearStart, yearEnd + 1):
warmpool = None
wnodata = None
maxwarm = None
minWarm = None
print "Processing year {0}".format(iyear)
for imonth in range(1, 12 + 1):
idate = "{0}{1:02}".format(iyear, imonth)
fname = "{0}/{1}_{2}.nc".format(sstdir, sstrootname, idate)
thisFile = cdms2.open(fname)
thisVar = numpy.ravel(thisFile[variable][:])
if latWindow is not None:
if latWindowMatrix is None:
latWindowMatrix = numpy.zeros(thisFile[variable].shape)
for ii in xrange(latWindow[0], latWindow[1] + 1):
latWindowMatrix[:, ii] = 1
thisVarTmp = thisVar
thisVar = numpy.multiply(thisVarTmp, numpy.ravel(latWindowMatrix))
if warmpool is None:
dimVar = numpy.squeeze(thisFile[variable][:]).shape
varUnits = thisFile[variable].units
warmpool = numpy.ravel(numpy.zeros(dimVar))
wnodata = thisVar >= nodata
maxWarm = numpy.ravel(numpy.zeros(dimVar))
minWarm = numpy.ravel(numpy.zeros(dimVar))
monthMin = numpy.ravel(numpy.zeros(dimVar))
monthMax = numpy.ravel(numpy.zeros(dimVar))
# on first iteration, no comparison to the previous state (warmpool>=threshold)
wwp = (thisVar >= threshold) * (thisVar < nodata)
wmax = (thisVar < nodata) * (thisVar > nodata) # set to false, everywhere
wmin = (thisVar < nodata) * (thisVar > nodata) # set to False, everywhere
maxWarm[wwp] = thisVar[wwp]
minWarm[wwp] = thisVar[wwp]
monthMax[wwp] = imonth
monthMin[wwp] = imonth
else:
# warmpool: for all months, temperature > threshold
# means that current value AND memo value are > threshold
wwp = (thisVar >= threshold) * (warmpool >= threshold) * (thisVar < nodata)
wmax = (thisVar >= threshold) * (warmpool >= threshold) * (thisVar < nodata) * (thisVar >= warmpool)
wmin = (thisVar >= threshold) * (warmpool >= threshold) * (thisVar < nodata) * (thisVar < warmpool)
# reset warmpool to 0, to keep only the minimal extension
# we will encode the max and min observed
if wwp.any():
maxWarm[:] = 0
minWarm[:] = 0
maxWarm[wwp] = warmpool[wwp]
if wmax.any():
maxWarm[wmax] = thisVar[wmax]
monthMax[wmax] = imonth
minWarm[wwp] = warmpool[wwp]
if wmin.any():
minWarm[wmin] = thisVar[wmin]
monthMin[wmin] = imonth
warmpool[:] = 0 # reset warmpool to keep intersection between iterations
warmpool[wwp] = thisVar[wwp]
else:
warmpool[:] = 0
thisFile.close()
# ensure mask is set
if wnodata.any():
warmpool[wnodata] = nodata
maxWarm[wnodata] = nodata
minWarm[wnodata] = nodata
monthMin[wnodata] = nodata
monthMax[wnodata] = nodata
wtonull = warmpool == 0
if wtonull.any():
monthMin[wtonull] = 0
monthMax[wtonull] = 0
warea = (warmpool >= threshold) * (warmpool < nodata)
surface = EarthSurface * numpy.ravel(weights)[warea].sum()
areaWP.append([iyear, surface])
wpOut = cdms2.createVariable(
warmpool.reshape(dimVar),
typecode="f",
id="warmpool",
fill_value=nodata,
grid=referenceGrid,
copyaxes=0,
attributes=dict(long_name="warmpool, all temperatures method, year {0}".format(iyear), units=varUnits),
)
wpMax = cdms2.createVariable(
maxWarm.reshape(dimVar),
typecode="f",
id="warmpool_max",
fill_value=nodata,
grid=referenceGrid,
copyaxes=0,
attributes=dict(long_name="warmpool max temperature, year {0}".format(iyear), units=varUnits),
)
wpMin = cdms2.createVariable(
minWarm.reshape(dimVar),
typecode="f",
id="warmpool_min",
fill_value=nodata,
grid=referenceGrid,
copyaxes=0,
attributes=dict(long_name="warmpool min temperature, year {0}".format(iyear), units=varUnits),
)
monthMin = cdms2.createVariable(
monthMin.reshape(dimVar),
typecode="i",
id="min_date",
fill_value=0,
grid=referenceGrid,
copyaxes=0,
attributes=dict(long_name="warmpool month of min(1-12), year {0}".format(iyear), units=varUnits),
)
monthMax = cdms2.createVariable(
monthMax.reshape(dimVar),
typecode="i",
id="max_date",
fill_value=0,
grid=referenceGrid,
copyaxes=0,
attributes=dict(long_name="warmpool month of max(1-12), year {0}".format(iyear), units=varUnits),
)
outfilename = "{0}/warmpool_{1}.nc".format(outdir, iyear)
if os.path.exists(outfilename):
os.remove(outfilename)
outfile = cdms2.open(outfilename, "w")
outfile.write(wpOut)
outfile.write(wpMax)
outfile.write(wpMin)
outfile.write(monthMin)
outfile.write(monthMax)
outfile.close()
return areaWP
newv.stash_item = var.stash_item[0]
newv.missing_value = missval
newv._FillValue = missval
try:
newv.units = var.units
except AttributeError:
pass
file.history += "\n%s: Processed %s" % (datetime.datetime.today().strftime('%Y-%m-%d %H:%M'), ifile)
# Get appropriate file position
# Uses 360 day calendar, all with same base time so must be 30 days on.
k = len(newtime)
# float needed here to get the later logical tests to work properly
avetime = float(MV.average(timevals[:])) # Works in either case
if k>0:
if average:
#if newtime[-1] != (avetime - 30):
# For Gregorian calendar relax this a bit
# Sometimes get differences slightly > 31
if not 28 <= avetime - newtime[-1] <= 31.5:
raise error, "Times not consecutive %f %f %f" % (newtime[-1], avetime, timevals[0])
else:
if k > 1:
# Need a better test that works when k = 1. This is just a
# temporary workaround
# For monthly data
if 27 < newtime[-1] - newtime[-2] < 32:
if not 27 < timevals[0] - newtime[-1] < 32:
raise error, "Monthly times not consecutive %f %f " % (newtime[-1], timevals[0])
