def shiftGrid(infile, outfile, variable, latShift=0, lonShift=-280): if os.path.exists(outfile): os.remove(outfile) thisfile=cdms2.open(infile) thisLon = thisfile[variable].getLongitude() thisLon[:].data[:] = (thisLon[:].data[:] - lonShift)%360 newvar = cdms2.createVariable(MV.array(thisfile[variable][:]), id=variable, fill_value=1.e20) newvar.setAxisList((thisfile[variable].getTime(), thisfile[variable].getLatitude(), thisLon)) if os.path.exists(outfile): os.remove(outfile) outFile = cdms2.open(outfile,'w') outFile.write(newvar) thisfile.close() outFile.close() sys.exit(1)
#------------------------------------------------------------------------------------------------ # Create canvas --- canvas = vcs.init(geometry=(1200,800)) canvas.open() template = canvas.createtemplate() template.blank(['title','mean','min','max','dataname','crdate','crtime','units']) ## Turn off additional information #template.list() ## This commend could give list of items # Set ploting range --- lat1=-60 lat2=80 lon1=0 lon2=360 # Wind speed --- data['wnd'] = MV.sqrt(data['ua']**2+data['va']**2) iso = canvas.createisofill() iso.datawc_x1 = lon1 iso.datawc_x2 = lon2 iso.datawc_y1 = lat1 iso.datawc_y2 = lat2 canvas.setcolormap('blue_to_orange') canvas.plot(data['wnd'],iso,template) # Geopotential height field --- lines1 = vcs.createisoline() lines1.datawc_x1 = lon1 lines1.datawc_x2 = lon2 lines1.datawc_y1 = lat1 lines1.datawc_y2 = lat2 lines1.linecolors = ['black']
def process_file(ifile,suffix,average=False,forcedaily=False,mask=True,xlist=[]): try: d = cdms2.open(ifile) except: print "Error opening file", ifile usage() sys.exit(1) hcrit = 0.5 # Critical value of Heavyside function for inclusion. ofilelist = [] for vn in d.variables: var = d.variables[vn] # Need to check whether it really has a stash_item to skip coordinate variables # Note: need to match both item and section number if not hasattr(var,'stash_item'): continue item_code = var.stash_section[0]*1000 + var.stash_item[0] if item_code in xlist: print "Skipping", item_code continue grid = var.getGrid() time = var.getTime() timevals = np.array(time[:]) if forcedaily: # Work around cdms error in times for k in range(len(time)): timevals[k] = round(timevals[k],1) umvar = stashvar.StashVar(item_code,var.stash_model[0]) vname = umvar.name print vname, var[0,0,0,0] # Create filename from variable name and cell_methods, # checking for name collisions if suffix: ofile = "%s_%s.nc" % (umvar.uniquename, suffix) else: ofile = "%s.nc" % umvar.uniquename if ofile in ofilelist: raise Exception("Duplicate file name %s" % ofile) ofilelist.append(ofile) # If output file exists then append to it, otherwise create a new file try: file = cdms2.openDataset(ofile, 'r+') newv = file.variables[vname] newtime = newv.getTime() except cdms2.error.CDMSError: file = cdms2.createDataset(ofile) # Stop it creating the bounds_latitude, bounds_longitude variables cdms2.setAutoBounds("off") # By default get names like latitude0, longitude1 # Need this awkwardness to get the variable/dimension name set correctly # Is there a way to change the name cdms uses after # newlat = newgrid.getLatitude() ???? newlat = file.createAxis('lat', grid.getLatitude()[:]) newlat.standard_name = "latitude" newlat.axis = "Y" newlat.units = 'degrees_north' newlon = file.createAxis('lon', grid.getLongitude()[:]) newlon.standard_name = "longitude" newlon.axis = "X" newlon.units = 'degrees_east' order = var.getOrder() if order[1] == 'z': lev = var.getLevel() if len(lev) > 1: newlev = file.createAxis('lev', lev[:]) for attr in ('standard_name', 'units', 'positive', 'axis'): if hasattr(lev,attr): setattr(newlev, attr, getattr(lev,attr)) else: newlev = None else: # Pseudo-dimension pdim = var.getAxis(1) if len(pdim) > 1: newlev = file.createAxis('pseudo', pdim[:]) else: newlev = None newtime = file.createAxis('time', None, cdms2.Unlimited) newtime.standard_name = "time" newtime.units = time.units # "days since " + `baseyear` + "-01-01 00:00" newtime.setCalendar(time.getCalendar()) newtime.axis = "T" if var.dtype == np.dtype('int32'): vtype = cdms2.CdInt missval = -2147483647 else: vtype = cdms2.CdFloat missval = 1.e20 if newlev: newv = file.createVariable(vname, vtype, (newtime, newlev, newlat, newlon)) else: newv = file.createVariable(vname, vtype, (newtime, newlat, newlon)) for attr in ("standard_name", "long_name", "units"): if hasattr(umvar, attr): newv.setattribute(attr, getattr(umvar,attr)) newv.missing_value = missval newv.stash_section=var.stash_section[0] newv.stash_item=var.stash_item[0] newv._FillValue = missval try: newv.units = var.units except AttributeError: pass # Get appropriate file position # Uses 360 day calendar, all with same base time so must be 30 days on. k = len(newtime) # float needed here to get the later logical tests to work properly avetime = float(MV.average(timevals[:])) # Works in either case if k>0: if average: # if newtime[-1] != (avetime - 30): # For Gregorian calendar relax this a bit # Sometimes get differences slightly > 31 if not 28 <= avetime - newtime[-1] <= 31.5: raise error, "Times not consecutive %f %f %f" % (newtime[-1], avetime, timevals[0]) else: if k > 1: # Need a better test that works when k = 1. This is just a # temporary workaround if not np.allclose( newtime[-1] + (newtime[-1]-newtime[-2]), timevals[0] ): raise error, "Times not consecutive %f %f " % (newtime[-1], timevals[0]) if (30201 <= item_code <= 30303) and mask: # P LEV/UV GRID with missing values treated as zero. # Needs to be corrected by Heavyside fn heavyside = d.variables['psag'] # Check variable code as well as the name. if heavyside.stash_item[0] != 301 or heavyside.stash_section[0] != 30: raise error, "Heavyside variable code mismatch" if average: newtime[k] = avetime if var.shape[1] > 1: # multiple levels newv[k] = MV.average(var[:],axis=0).astype(np.float32) else: # single level newv[k] = MV.average(var[:],axis=0)[0].astype(np.float32) else: for i in range(len(timevals)): if var.shape[1] > 1: # Multi-level if (30201 <= item_code <= 30303) and mask: newv[k+i] = np.where( np.greater(heavyside[i], hcrit), var[i]/heavyside[0], newv.getMissing()) else: newv[k+i] = var[i] else: newv[k+i] = var[i,0] newtime[k+i] = timevals[i] file.close()
canvas = vcs.init(geometry=(1200, 800)) canvas.open() template = canvas.createtemplate() template.blank( ['title', 'mean', 'min', 'max', 'dataname', 'crdate', 'crtime', 'units']) ## Turn off additional information #template.list() ## This commend could give list of items # Set ploting range --- lat1 = -60 lat2 = 80 lon1 = 0 lon2 = 360 # Wind speed --- data['wnd'] = MV.sqrt(data['ua']**2 + data['va']**2) iso = canvas.createisofill() iso.datawc_x1 = lon1 iso.datawc_x2 = lon2 iso.datawc_y1 = lat1 iso.datawc_y2 = lat2 canvas.setcolormap('blue_to_orange') canvas.plot(data['wnd'], iso, template) # Geopotential height field --- lines1 = vcs.createisoline() lines1.datawc_x1 = lon1 lines1.datawc_x2 = lon2 lines1.datawc_y1 = lat1 lines1.datawc_y2 = lat2 lines1.linecolors = ['black']
def do_yearlyWPAvg(sstdir, sstrootname, variable, outdir, yearStart, yearEnd, threshold=28.5 + 273.15, latWindow=None): nodata = 1.0e20 varUnits = None thisFilter = numpy.ones((3, 3)) # thisFilter[1][1]=1 EarthSurface = 510072000 factor = 1 # (2*85*360) /( 360.0*180.0) areaWP = [] latWindowMatrix = None for iyear in range(yearStart, yearEnd + 1): tempAvg = None wdata = None weights = None dimVar = None counter = None print "Processing year {0}".format(iyear) for imonth in range(1, 12 + 1): idate = "{0}{1:02}".format(iyear, imonth) fname = "{0}/{1}_{2}.nc".format(sstdir, sstrootname, idate) thisFile = cdms2.open(fname) thisVar = numpy.ravel(thisFile[variable][:]) if latWindow is not None: if latWindowMatrix is None: latWindowMatrix = numpy.zeros(thisFile[variable].shape) for ii in xrange(latWindow[0], latWindow[1] + 1): latWindowMatrix[:, ii] = 1 thisVarTmp = thisVar thisVar = numpy.multiply(thisVarTmp, numpy.ravel(latWindowMatrix)) if tempAvg is None: # settings thisGrid = thisFile[variable].getGrid() if thisGrid is None: thisGrid = makeGrid() (latws, lonws) = thisGrid.getWeights() weights = MV.outerproduct(latws, lonws) wdata = thisVar < nodata tempAvg = numpy.zeros(thisVar.shape) tempAvg[wdata] = thisVar[wdata] dimVar = numpy.squeeze(thisFile[variable][:]).shape counter = numpy.zeros(tempAvg.shape, dtype="float") counter[wdata] = 1 else: wdata = thisVar < nodata counter[wdata] = counter[wdata] + 1 tempAvg[wdata] = tempAvg[wdata] + thisVar[wdata] # compute average wdivide = counter > 0 avg = numpy.zeros(tempAvg.shape) if wdivide.any: avg[wdivide] = tempAvg[wdivide] / counter[wdivide] # set to areas < threshold wtzero = avg < threshold avg[wtzero] = 0 # compute current area warea = (avg >= threshold) * (avg < nodata) area = factor * EarthSurface * numpy.ravel(weights)[warea].sum() areaWP.append([iyear, area]) # create variables outAreaTmp = numpy.reshape(avg, dimVar) # filter before saving: grid stiching area: less data for ensemble mean here outAreaBis = do_interp(outAreaTmp, threshold, nodata, 156, 156 + 3, 0, 4 * 80) # rebuild outArea = do_convolve(outAreaBis, thisFilter, threshold, nodata, 156 + 2, 156 + 5, 0, 4 * 80) # smoothen wpOut = cdms2.createVariable( outArea, typecode="f", id="warmpool", grid=thisGrid, copyaxes=1, attributes=dict(long_name="warmpool, average temperature method, year {0}".format(iyear), units=varUnits), ) # write to file outfilename = "{0}/warmpool_{1}.nc".format(outdir, iyear) if os.path.exists(outfilename): os.remove(outfilename) outfile = cdms2.open(outfilename, "w") outfile.write(wpOut) outfile.close() # close files thisFile.close() return areaWP
def do_yearlyWPall(sstdir, sstrootname, variable, outdir, yearStart, yearEnd, latWindow=None): threshold = 28 + 273.15 nodata = 1.0e20 varUnits = None referenceGrid = None referenceGrid = makeGrid() latws, lonwts = referenceGrid.getWeights() weights = MV.outerproduct(latws, lonwts) EarthSurface = 510072000 areaWP = [] latWindowMatrix = None for iyear in range(yearStart, yearEnd + 1): warmpool = None wnodata = None maxwarm = None minWarm = None print "Processing year {0}".format(iyear) for imonth in range(1, 12 + 1): idate = "{0}{1:02}".format(iyear, imonth) fname = "{0}/{1}_{2}.nc".format(sstdir, sstrootname, idate) thisFile = cdms2.open(fname) thisVar = numpy.ravel(thisFile[variable][:]) if latWindow is not None: if latWindowMatrix is None: latWindowMatrix = numpy.zeros(thisFile[variable].shape) for ii in xrange(latWindow[0], latWindow[1] + 1): latWindowMatrix[:, ii] = 1 thisVarTmp = thisVar thisVar = numpy.multiply(thisVarTmp, numpy.ravel(latWindowMatrix)) if warmpool is None: dimVar = numpy.squeeze(thisFile[variable][:]).shape varUnits = thisFile[variable].units warmpool = numpy.ravel(numpy.zeros(dimVar)) wnodata = thisVar >= nodata maxWarm = numpy.ravel(numpy.zeros(dimVar)) minWarm = numpy.ravel(numpy.zeros(dimVar)) monthMin = numpy.ravel(numpy.zeros(dimVar)) monthMax = numpy.ravel(numpy.zeros(dimVar)) # on first iteration, no comparison to the previous state (warmpool>=threshold) wwp = (thisVar >= threshold) * (thisVar < nodata) wmax = (thisVar < nodata) * (thisVar > nodata) # set to false, everywhere wmin = (thisVar < nodata) * (thisVar > nodata) # set to False, everywhere maxWarm[wwp] = thisVar[wwp] minWarm[wwp] = thisVar[wwp] monthMax[wwp] = imonth monthMin[wwp] = imonth else: # warmpool: for all months, temperature > threshold # means that current value AND memo value are > threshold wwp = (thisVar >= threshold) * (warmpool >= threshold) * (thisVar < nodata) wmax = (thisVar >= threshold) * (warmpool >= threshold) * (thisVar < nodata) * (thisVar >= warmpool) wmin = (thisVar >= threshold) * (warmpool >= threshold) * (thisVar < nodata) * (thisVar < warmpool) # reset warmpool to 0, to keep only the minimal extension # we will encode the max and min observed if wwp.any(): maxWarm[:] = 0 minWarm[:] = 0 maxWarm[wwp] = warmpool[wwp] if wmax.any(): maxWarm[wmax] = thisVar[wmax] monthMax[wmax] = imonth minWarm[wwp] = warmpool[wwp] if wmin.any(): minWarm[wmin] = thisVar[wmin] monthMin[wmin] = imonth warmpool[:] = 0 # reset warmpool to keep intersection between iterations warmpool[wwp] = thisVar[wwp] else: warmpool[:] = 0 thisFile.close() # ensure mask is set if wnodata.any(): warmpool[wnodata] = nodata maxWarm[wnodata] = nodata minWarm[wnodata] = nodata monthMin[wnodata] = nodata monthMax[wnodata] = nodata wtonull = warmpool == 0 if wtonull.any(): monthMin[wtonull] = 0 monthMax[wtonull] = 0 warea = (warmpool >= threshold) * (warmpool < nodata) surface = EarthSurface * numpy.ravel(weights)[warea].sum() areaWP.append([iyear, surface]) wpOut = cdms2.createVariable( warmpool.reshape(dimVar), typecode="f", id="warmpool", fill_value=nodata, grid=referenceGrid, copyaxes=0, attributes=dict(long_name="warmpool, all temperatures method, year {0}".format(iyear), units=varUnits), ) wpMax = cdms2.createVariable( maxWarm.reshape(dimVar), typecode="f", id="warmpool_max", fill_value=nodata, grid=referenceGrid, copyaxes=0, attributes=dict(long_name="warmpool max temperature, year {0}".format(iyear), units=varUnits), ) wpMin = cdms2.createVariable( minWarm.reshape(dimVar), typecode="f", id="warmpool_min", fill_value=nodata, grid=referenceGrid, copyaxes=0, attributes=dict(long_name="warmpool min temperature, year {0}".format(iyear), units=varUnits), ) monthMin = cdms2.createVariable( monthMin.reshape(dimVar), typecode="i", id="min_date", fill_value=0, grid=referenceGrid, copyaxes=0, attributes=dict(long_name="warmpool month of min(1-12), year {0}".format(iyear), units=varUnits), ) monthMax = cdms2.createVariable( monthMax.reshape(dimVar), typecode="i", id="max_date", fill_value=0, grid=referenceGrid, copyaxes=0, attributes=dict(long_name="warmpool month of max(1-12), year {0}".format(iyear), units=varUnits), ) outfilename = "{0}/warmpool_{1}.nc".format(outdir, iyear) if os.path.exists(outfilename): os.remove(outfilename) outfile = cdms2.open(outfilename, "w") outfile.write(wpOut) outfile.write(wpMax) outfile.write(wpMin) outfile.write(monthMin) outfile.write(monthMax) outfile.close() return areaWP
newv.stash_item = var.stash_item[0] newv.missing_value = missval newv._FillValue = missval try: newv.units = var.units except AttributeError: pass file.history += "\n%s: Processed %s" % (datetime.datetime.today().strftime('%Y-%m-%d %H:%M'), ifile) # Get appropriate file position # Uses 360 day calendar, all with same base time so must be 30 days on. k = len(newtime) # float needed here to get the later logical tests to work properly avetime = float(MV.average(timevals[:])) # Works in either case if k>0: if average: #if newtime[-1] != (avetime - 30): # For Gregorian calendar relax this a bit # Sometimes get differences slightly > 31 if not 28 <= avetime - newtime[-1] <= 31.5: raise error, "Times not consecutive %f %f %f" % (newtime[-1], avetime, timevals[0]) else: if k > 1: # Need a better test that works when k = 1. This is just a # temporary workaround # For monthly data if 27 < newtime[-1] - newtime[-2] < 32: if not 27 < timevals[0] - newtime[-1] < 32: raise error, "Monthly times not consecutive %f %f " % (newtime[-1], timevals[0])